Ticket #13960: filetransfers.diff

django/conf/global_settings.py

@@ -257 +257 @@
 # Default file storage mechanism that holds media.
 DEFAULT_FILE_STORAGE = 'django.core.files.storage.FileSystemStorage'
 
+FILE_BACKENDS = ()
+
 # Absolute path to the directory that holds media.
 # Example: "/home/media/media.lawrence.com/"
 MEDIA_ROOT = ''

new file django/core/files/transfers.py

@@ +1 @@
+from django.conf import settings
+from django.core.exceptions import ImproperlyConfigured
+from django.core.files.storage import default_storage
+from django.http import HttpResponse
+from django.utils.encoding import smart_str
+from django.utils.importlib import import_module
+import mimetypes
+
+class BaseFileBackend(object):
+    def __init__(self, model, fields):
+        self.model = model
+        if not hasattr(fields, '__iter__'):
+            fields = (fields,)
+        self.fields = fields
+
+    # Add a convenience accessor for serve, download_url,
+    # and get_storage_backend which only act on a single FileField
+    def _get_field(self):
+        if len(self.fields) != 1:
+            raise ValueError('Expected only a single FileField, got %d' % len(self.fields))
+        return self.fields[0]
+    field = property(_get_field)
+
+    def prepare_upload(self, request, url):
+        return None
+
+    def serve(self, request, file, save_as=False, content_type=None):
+        return None
+
+    def download_url(self, file):
+        return None
+
+    def get_storage_backend(self):
+        return None
+
+# Public API
+def prepare_upload(model, fields, request, url=None):
+    if not url:
+        url = request.get_full_path()
+    for backend in _get_backends(model, fields):
+        result = backend.prepare_upload(request, url)
+        if result is not None:
+            return result
+
+    # By default we simply return the URL unmodified
+    return url, {}
+
+def serve_file(model, fields, request, file, save_as=False, content_type=None):
+    filename = file.name.rsplit('/')[-1]
+    if save_as is True:
+        save_as = filename
+    if not content_type:
+        content_type = mimetypes.guess_type(filename)[0]
+    for backend in _get_backends(model, fields):
+        result = backend.serve(request, file, save_as=save_as, content_type=content_type)
+        if result is not None:
+            return result
+
+    return _default_serve_file(file, save_as, content_type)
+
+def download_url(model, fields, file):
+    for backend in _get_backends(model, fields):
+        result = backend.download_url(file)
+        if result is not None:
+            return result
+
+    # By default we use MEDIA_URL
+    return settings.MEDIA_URL + file.name
+
+def get_storage_backend(model, fields):
+    for backend in _get_backends(model, fields):
+        result = backend.get_storage_backend()
+        if result is not None:
+            return result
+
+    # By default there is no public download URL
+    return default_storage
+
+# Internal utilities
+class ChunkedFile(object):
+    def __init__(self, file):
+        self.file = file
+
+    def __iter__(self):
+        return self.file.chunks()
+
+def _default_serve_file(file, save_as, content_type):    
+    """
+    Serves the file in chunks for efficiency reasons.
+    
+    The transfer still goes through Django itself, so it's much worse than
+    using the web server, but at least it works with all configurations.
+    """
+    response = HttpResponse(ChunkedFile(file), content_type=content_type)
+    if save_as:
+        response['Content-Disposition'] = smart_str(u'attachment; filename=%s' % save_as)
+    if file.size is not None:
+        response['Content-Length'] = file.size
+    return response
+
+def _get_backends(model, fields):
+    return (_load_backend(name, model, fields)
+            for name in settings.FILE_BACKENDS)
+
+def _load_backend(path, model, fields):
+    module_name, attr_name = path.rsplit('.', 1)
+    try:
+        mod = import_module(module_name)
+    except ImportError, e:
+        raise ImproperlyConfigured('Error importing file backend module %s: "%s"' % (module_name, e))
+    except ValueError, e:
+        raise ImproperlyConfigured('Error importing file backend module. Is FILE_BACKENDS a correctly defined list or tuple?')
+    try:
+        backend = getattr(mod, attr_name)
+    except AttributeError:
+        raise ImproperlyConfigured('Module "%s" does not define a "%s" file backend' % (module_name, attr_name))
+    return backend(model, fields)

django/core/files/uploadhandler.py

@@ -84 +84 @@
         """
         pass
 
-    def new_file(self, field_name, file_name, content_type, content_length, charset=None):
+    def new_file(self, field_name, file_name, content_type, content_length,  charset=None, content_type_extra=None):
         """
         Signal that a new file has been started.
 
@@ -96 +96 @@
         self.content_type = content_type
         self.content_length = content_length
         self.charset = charset
+        if content_type_extra is None:
+            content_type_extra = {}
+        self.content_type_extra = content_type_extra
 
     def receive_data_chunk(self, raw_data, start):
         """

django/db/models/fields/files.py

@@ -6 +6 @@
 from django.conf import settings
 from django.db.models.fields import Field
 from django.core.files.base import File, ContentFile
-from django.core.files.storage import default_storage
 from django.core.files.images import ImageFile, get_image_dimensions
+from django.core.files.transfers import prepare_upload, serve_file, download_url, get_storage_backend
 from django.core.files.uploadedfile import UploadedFile
 from django.utils.functional import curry
 from django.db.models import signals
@@ -139 +139 @@
         # be restored later, by FileDescriptor below.
         return {'name': self.name, 'closed': False, '_committed': True, '_file': None}
 
+    def serve(self, request, save_as=False, content_type=None):
+        self._require_file()
+        return serve_file(self.field.model, self.field, request, self, save_as=save_as, content_type=content_type)
+
+    def download_url(self):
+        self._require_file()
+        return download_url(self.field.model, self.field, self)
+
 class FileDescriptor(object):
     """
     The descriptor for the file attribute on the model instance. Returns a
@@ -156 +164 @@
 
     def __get__(self, instance=None, owner=None):
         if instance is None:
-            raise AttributeError(
-                "The '%s' attribute can only be accessed from %s instances."
-                % (self.field.name, owner.__name__))
+            return self.field
 
         # This is slightly complicated, so worth an explanation.
         # instance.file`needs to ultimately return some instance of `File`,
@@ -224 +230 @@
             if arg in kwargs:
                 raise TypeError("'%s' is not a valid argument for %s." % (arg, self.__class__))
 
-        self.storage = storage or default_storage
+        self.storage = storage
         self.upload_to = upload_to
         if callable(upload_to):
             self.generate_filename = upload_to
@@ -257 +263 @@
 
     def contribute_to_class(self, cls, name):
         super(FileField, self).contribute_to_class(cls, name)
+        if self.storage is None:
+            self.storage = get_storage_backend(self.model, self)
         setattr(cls, self.name, self.descriptor_class(self))
         signals.post_delete.connect(self.delete_file, sender=cls)
 
@@ -297 +305 @@
         defaults.update(kwargs)
         return super(FileField, self).formfield(**defaults)
 
+    def prepare_upload(self, request, url=None):
+        return prepare_upload(self.model, self, request, url=url)
+
 class ImageFileDescriptor(FileDescriptor):
     """
     Just like the FileDescriptor, but for ImageFields. The only difference is

django/forms/models.py

@@ -3 +3 @@
 and database field objects.
 """
 
+from django.core.files.transfers import prepare_upload
 from django.db import connections
 from django.utils.encoding import smart_unicode, force_unicode
 from django.utils.datastructures import SortedDict
@@ -231 +232 @@
     def __init__(self, data=None, files=None, auto_id='id_%s', prefix=None,
                  initial=None, error_class=ErrorList, label_suffix=':',
                  empty_permitted=False, instance=None):
+        self._file_upload_data = {}
+
         opts = self._meta
         if instance is None:
             if opts.model is None:
@@ -372 +375 @@
 
     save.alters_data = True
 
+    def prepare_upload(self, request, url=None):
+        from django.db import models
+        for name in self._file_upload_data:
+            del self.fields[name]
+
+        file_fields = []
+        for name in self.fields:
+            try:
+                field = self._meta.model._meta.get_field(name)
+                if isinstance(field, models.FileField):
+                    file_fields.append(field)
+            except models.FieldDoesNotExist:
+                pass
+
+        result = prepare_upload(self._meta.model, file_fields, request, url=url)
+        self.upload_url, self._file_upload_data = result
+
+        for name, value in self._file_upload_data.items():
+            self.fields[name] = Field(initial=value, required=False, widget=HiddenInput)
+
 class ModelForm(BaseModelForm):
     __metaclass__ = ModelFormMetaclass
 

django/http/multipartparser.py

@@ -169 +169 @@
                     file_name = self.IE_sanitize(unescape_entities(file_name))
 
                     content_type = meta_data.get('content-type', ('',))[0].strip()
+                    content_type_extra = meta_data.get('content-type', (0,{}))[1]
                     try:
-                        charset = meta_data.get('content-type', (0,{}))[1].get('charset', None)
+                        charset = content_type_extra.get('charset', None)
                     except:
                         charset = None
 
@@ -185 +186 @@
                             try:
                                 handler.new_file(field_name, file_name,
                                                  content_type, content_length,
-                                                 charset)
+                                                 charset, content_type_extra)
                             except StopFutureHandlers:
                                 break
 

tests/modeltests/files/models.py

@@ -34 +34 @@
 __test__ = {'API_TESTS':"""
 # Attempting to access a FileField from the class raises a descriptive error
 >>> Storage.normal
-Traceback (most recent call last):
-...
-AttributeError: The 'normal' attribute can only be accessed from Storage instances.
+<django.db.models.fields.files.FileField object at ...>
 
 # An object without a file has limited functionality.
 

new file tests/regressiontests/file_uploads/backends.py

@@ +1 @@
+from django.conf import settings
+from django.core.files.transfers import BaseFileBackend
+from django.http import HttpResponseRedirect
+import models
+
+class TestFileBackend(BaseFileBackend):
+    def prepare_upload(self, request, url, *args, **kwargs):
+        if self.model.__module__ == models.__name__:
+            return '/redirect' + url, {'x-extra': 'abcd', 'x-data': 'efg'}
+
+    def serve(self, request, file, *args, **kwargs):
+        if self.model.__module__ == models.__name__ and not file.name.endswith('pass'):
+            return HttpResponseRedirect('/download/')
+
+    def download_url(self, file):
+        if self.model.__module__ == models.__name__ and not file.name.endswith('pass'):
+            return '/public/'
+
+    def get_storage_backend(self, *args, **kwargs):
+        if self.model.__module__ == models.__name__:
+            return models.temp_storage

new file tests/regressiontests/file_uploads/forms.py

@@ +1 @@
+from django import forms
+from models import FileModel
+
+class FileForm(forms.ModelForm):
+    class Meta:
+        model = FileModel

tests/regressiontests/file_uploads/models.py

@@ -1 +1 @@
 import tempfile
 import os
+from django.conf import settings
 from django.db import models
 from django.core.files.storage import FileSystemStorage
 
 temp_storage = FileSystemStorage(tempfile.mkdtemp())
 UPLOAD_TO = os.path.join(temp_storage.location, 'test_upload')
+settings.FILE_BACKENDS = ('%s.backends.TestFileBackend' % __name__.rsplit('.', 1)[0],) + \
+    settings.FILE_BACKENDS
 
 class FileModel(models.Model):
-    testfile = models.FileField(storage=temp_storage, upload_to='test_upload')
+    testfile = models.FileField(upload_to='test_upload')

tests/regressiontests/file_uploads/tests.py

@@ -5 +5 @@
 import unittest
 from StringIO import StringIO
 
+from django.conf import settings
 from django.core.files import temp as tempfile
 from django.core.files.uploadedfile import SimpleUploadedFile
 from django.test import TestCase, client
@@ -17 +18 @@
 
 UNICODE_FILENAME = u'test-0123456789_中文_Orléans.jpg'
 
+class FileBackendTests(TestCase):
+    def setUp(self):
+        if not os.path.isdir(temp_storage.location):
+            os.makedirs(temp_storage.location)
+        if os.path.isdir(UPLOAD_TO):
+            shutil.rmtree(UPLOAD_TO)
+
+    def tearDown(self):
+        shutil.rmtree(temp_storage.location)
+
+    def test_prepare_upload(self):
+        response = self.client.get('/file_uploads/prepare_upload/')
+        self.assertEqual(response.status_code, 200)
+        data = simplejson.loads(response.content)
+        self.assertEqual(data, [
+            '/redirect/file_uploads/prepare_upload/', [
+                '<input type="file" name="testfile" id="id_testfile" />',
+                '<input type="hidden" name="x-data" value="efg" id="id_x-data" />',
+                '<input type="hidden" name="x-extra" value="abcd" id="id_x-extra" />',
+            ]
+        ])
+
+    def test_serve_file(self):
+        obj = FileModel()
+        obj.testfile.save('foo.txt', SimpleUploadedFile('foo.txt', 'x'))
+        obj.save()
+        response = self.client.get('/file_uploads/serve/')
+        self.assertEqual(response.status_code, 302)
+        self.assertEqual(response['Location'], 'http://testserver/download/')
+
+    def test_serve_file_fallback(self):
+        obj = FileModel()
+        obj.testfile.save('foo.pass', SimpleUploadedFile('foo.pass', 'x'))
+        obj.save()
+        response = self.client.get('/file_uploads/serve/')
+        self.assertEqual(response.status_code, 200)
+
+    def test_download_url(self):
+        obj = FileModel()
+        obj.testfile.save('foo2.txt', SimpleUploadedFile('foo2.txt', 'x'))
+        obj.save()
+        url = obj.testfile.download_url()
+        self.assertEqual(url, '/public/')
+
+    def test_download_url_fallback(self):
+        obj = FileModel()
+        obj.testfile.save('foo2.pass', SimpleUploadedFile('foo2.pass', 'x'))
+        obj.save()
+        url = obj.testfile.download_url()
+        self.assertEqual(url, settings.MEDIA_URL + obj.testfile.name)
+
 class FileUploadTests(TestCase):
     def test_simple_upload(self):
         post_data = {
@@ -122 +174 @@
         response = self.client.request(**r)
 
         # The filenames should have been sanitized by the time it got to the view.
-        recieved = simplejson.loads(response.content)
+        received = simplejson.loads(response.content)
         for i, name in enumerate(scary_file_names):
-            got = recieved["file%s" % i]
+            got = received["file%s" % i]
             self.assertEqual(got, "hax0rd.txt")
 
     def test_filename_overflow(self):

tests/regressiontests/file_uploads/urls.py

@@ -10 +10 @@
     (r'^quota/broken/$',    views.file_upload_quota_broken),
     (r'^getlist_count/$',   views.file_upload_getlist_count),
     (r'^upload_errors/$',   views.file_upload_errors),
+    (r'^prepare_upload/$',  views.file_prepare_upload),
+    (r'^serve/$', views.file_serve),
 )

tests/regressiontests/file_uploads/views.py

@@ -2 +2 @@
 from django.core.files.uploadedfile import UploadedFile
 from django.http import HttpResponse, HttpResponseServerError
 from django.utils import simplejson
+from forms import FileForm
 from models import FileModel, UPLOAD_TO
 from uploadhandler import QuotaUploadHandler, ErroringUploadHandler
 from django.utils.hashcompat import sha_constructor
@@ -112 +113 @@
 def file_upload_errors(request):
     request.upload_handlers.insert(0, ErroringUploadHandler())
     return file_upload_echo(request)
+
+def file_prepare_upload(request):
+    form = FileForm()
+    form.prepare_upload(request)
+    return HttpResponse(simplejson.dumps([form.upload_url, list(map(unicode, form))]))
+
+def file_serve(request):
+    return FileModel.objects.all()[0].testfile.serve(request)