Ticket #13960: filetransfers.3.diff

django/conf/global_settings.py

@@ -258 +258 @@
 # Default file storage mechanism that holds media.
 DEFAULT_FILE_STORAGE = 'django.core.files.storage.FileSystemStorage'
 
+# High-level transfer backends
+FILE_TRANSFER_BACKENDS = (
+    'django.core.files.transfers.DefaultFileBackend',
+)
+
+# The default transfer backend's configuration mapping transfer_id to
+# backend name
+DEFAULT_FILE_TRANSFER_BACKENDS = {}
+
 # Absolute path to the directory that holds media.
 # Example: "/home/media/media.lawrence.com/"
 MEDIA_ROOT = ''

new file django/core/files/transfers.py

@@ +1 @@
+from django.conf import settings
+from django.core.exceptions import ImproperlyConfigured
+from django.core.files.storage import default_storage
+from django.http import HttpResponse
+from django.utils.encoding import smart_str
+from django.utils.importlib import import_module
+import mimetypes
+
+class BaseFileBackend(object):
+    def __init__(self, transfer_id):
+        self.transfer_id = transfer_id
+
+    def prepare_upload(self, request, url):
+        return None
+
+    def serve(self, request, file, save_as=False, content_type=None):
+        return None
+
+    def download_url(self, file):
+        return None
+
+    def get_storage_backend(self):
+        return None
+
+class DefaultFileBackend(BaseFileBackend):
+    def _get_backend_path(self):
+        if self.transfer_id in settings.DEFAULT_FILE_TRANSFER_BACKENDS:
+            return settings.DEFAULT_FILE_TRANSFER_BACKENDS[self.transfer_id]
+        parts = self.transfer_id.split('.')[:-1]
+        while parts:
+            name = '.'.join(parts) + '.*'
+            if name in settings.DEFAULT_FILE_TRANSFER_BACKENDS:
+                return settings.DEFAULT_FILE_TRANSFER_BACKENDS[name]
+            del parts[-1]
+        return None
+
+    def _get_backend(self):
+        path = self._get_backend_path()
+        if path:
+            return load_backend(path, self.transfer_id)
+        return None
+
+    def prepare_upload(self, *args, **kwargs):
+        backend = self._get_backend()
+        if backend:
+            return backend.prepare_upload(*args, **kwargs)
+        return None
+
+    def serve(self, *args, **kwargs):
+        backend = self._get_backend()
+        if backend:
+            return backend.serve(*args, **kwargs)
+        return None
+
+    def download_url(self, *args, **kwargs):
+        backend = self._get_backend()
+        if backend:
+            return backend.download_url(*args, **kwargs)
+        return None
+
+    def get_storage_backend(self, *args, **kwargs):
+        backend = self._get_backend()
+        if backend:
+            return backend.get_storage_backend(*args, **kwargs)
+        return None
+
+# Public API
+def prepare_upload(transfer_id, request, url=None):
+    if not url:
+        url = request.get_full_path()
+    for backend in _get_backends(transfer_id):
+        result = backend.prepare_upload(request, url)
+        if result is not None:
+            return result
+
+    # By default we simply return the URL unmodified
+    return url, {}
+
+def serve_file(transfer_id, request, file, save_as=False, content_type=None):
+    filename = file.name.rsplit('/')[-1]
+    if save_as is True:
+        save_as = filename
+    if not content_type:
+        content_type = mimetypes.guess_type(filename)[0]
+    for backend in _get_backends(transfer_id):
+        result = backend.serve(request, file, save_as=save_as, content_type=content_type)
+        if result is not None:
+            return result
+
+    return _default_serve_file(file, save_as, content_type)
+
+def download_url(transfer_id, file):
+    for backend in _get_backends(transfer_id):
+        result = backend.download_url(file)
+        if result is not None:
+            return result
+
+    # By default we use MEDIA_URL
+    return settings.MEDIA_URL + file.name
+
+def get_storage_backend(transfer_id):
+    for backend in _get_backends(transfer_id):
+        result = backend.get_storage_backend()
+        if result is not None:
+            return result
+
+    # By default there is no public download URL
+    return default_storage
+
+# Internal utilities
+class ChunkedFile(object):
+    def __init__(self, file):
+        self.file = file
+
+    def __iter__(self):
+        return self.file.chunks()
+
+def _default_serve_file(file, save_as, content_type):    
+    """
+    Serves the file in chunks for efficiency reasons.
+    
+    The transfer still goes through Django itself, so it's much worse than
+    using the web server, but at least it works with all configurations.
+    """
+    response = HttpResponse(ChunkedFile(file), content_type=content_type)
+    if save_as:
+        response['Content-Disposition'] = smart_str(u'attachment; filename=%s' % save_as)
+    if file.size is not None:
+        response['Content-Length'] = file.size
+    return response
+
+def _get_backends(*args, **kwargs):
+    return (load_backend(name, *args, **kwargs)
+            for name in settings.FILE_TRANSFER_BACKENDS)
+
+def load_backend(path, *args, **kwargs):
+    module_name, attr_name = path.rsplit('.', 1)
+    try:
+        mod = import_module(module_name)
+    except ImportError, e:
+        raise ImproperlyConfigured('Error importing file backend module %s: "%s"' % (module_name, e))
+    except ValueError, e:
+        raise ImproperlyConfigured('Error importing file backend module. Is FILE_TRANSFER_BACKENDS a correctly defined list or tuple?')
+    try:
+        backend = getattr(mod, attr_name)
+    except AttributeError:
+        raise ImproperlyConfigured('Module "%s" does not define a "%s" file backend' % (module_name, attr_name))
+    return backend(*args, **kwargs)

django/core/files/uploadhandler.py

@@ -84 +84 @@
         """
         pass
 
-    def new_file(self, field_name, file_name, content_type, content_length, charset=None):
+    def new_file(self, field_name, file_name, content_type, content_length,  charset=None, content_type_extra=None):
         """
         Signal that a new file has been started.
 
@@ -96 +96 @@
         self.content_type = content_type
         self.content_length = content_length
         self.charset = charset
+        if content_type_extra is None:
+            content_type_extra = {}
+        self.content_type_extra = content_type_extra
 
     def receive_data_chunk(self, raw_data, start):
         """

django/db/models/fields/files.py

@@ -6 +6 @@
 from django.conf import settings
 from django.db.models.fields import Field
 from django.core.files.base import File, ContentFile
-from django.core.files.storage import default_storage
 from django.core.files.images import ImageFile, get_image_dimensions
+from django.core.files.transfers import serve_file, download_url, get_storage_backend
 from django.core.files.uploadedfile import UploadedFile
 from django.utils.functional import curry
 from django.db.models import signals
@@ -139 +139 @@
         # be restored later, by FileDescriptor below.
         return {'name': self.name, 'closed': False, '_committed': True, '_file': None}
 
+    def serve(self, request, save_as=False, content_type=None):
+        self._require_file()
+        return serve_file(self.field.transfer_id, request, self, save_as=save_as, content_type=content_type)
+
+    def download_url(self):
+        self._require_file()
+        return download_url(self.field.transfer_id, self)
+
 class FileDescriptor(object):
     """
     The descriptor for the file attribute on the model instance. Returns a
@@ -219 +227 @@
 
     description = ugettext_lazy("File path")
 
-    def __init__(self, verbose_name=None, name=None, upload_to='', storage=None, **kwargs):
+    def __init__(self, verbose_name=None, name=None, upload_to='', storage=None, transfer_id=None, **kwargs):
         for arg in ('primary_key', 'unique'):
             if arg in kwargs:
                 raise TypeError("'%s' is not a valid argument for %s." % (arg, self.__class__))
 
-        self.storage = storage or default_storage
+        self.transfer_id = transfer_id
+        self.storage = storage
         self.upload_to = upload_to
         if callable(upload_to):
             self.generate_filename = upload_to
@@ -257 +266 @@
 
     def contribute_to_class(self, cls, name):
         super(FileField, self).contribute_to_class(cls, name)
+        if self.transfer_id is None:
+            self.transfer_id = '%s.%s.%s' % (cls.__module__,
+                                             cls.__name__,
+                                             self.name)
+        if self.storage is None:
+            self.storage = get_storage_backend(self.transfer_id)
         setattr(cls, self.name, self.descriptor_class(self))
         signals.post_delete.connect(self.delete_file, sender=cls)
 

django/forms/forms.py

@@ -3 +3 @@
 """
 
 from django.core.exceptions import ValidationError
+from django.core.files.transfers import prepare_upload
 from django.utils.copycompat import deepcopy
 from django.utils.datastructures import SortedDict
 from django.utils.html import conditional_escape
@@ -10 +11 @@
 from django.utils.safestring import mark_safe
 
 from fields import Field, FileField
-from widgets import Media, media_property, TextInput, Textarea
+from widgets import Media, media_property, TextInput, Textarea, HiddenInput
 from util import flatatt, ErrorDict, ErrorList
 
-__all__ = ('BaseForm', 'Form')
+__all__ = ('BaseForm', 'Form', 'RequestForm')
 
 NON_FIELD_ERRORS = '__all__'
 
@@ -386 +387 @@
     # BaseForm itself has no way of designating self.fields.
     __metaclass__ = DeclarativeFieldsMetaclass
 
+class BaseRequestFormTraits(object):
+    def __init__(self, request, **kwargs):
+        self.request = request
+        self._url = kwargs.pop('url', request.get_full_path())
+        if request.method == 'POST':
+            kwargs.setdefault('data', request.POST)
+            kwargs.setdefault('files', request.FILES)
+        self._file_upload_data = {}
+        super(BaseRequestFormTraits, self).__init__(**kwargs)
+        self.transfer_id = '%s.%s' % (self.__class__.__module__,
+                                      self.__class__.__name__)
+        self.prepare_upload()
+
+    def prepare_upload(self):
+        for name in self._file_upload_data:
+            del self.fields[name]
+
+        result = prepare_upload(self.transfer_id, self.request, url=self._url)
+        self.upload_url, self._file_upload_data = result
+
+        for name, value in self._file_upload_data.items():
+            self.fields[name] = Field(initial=value, required=False, widget=HiddenInput)
+
+class RequestForm(BaseRequestFormTraits, BaseForm):
+    __metaclass__ = DeclarativeFieldsMetaclass
+
 class BoundField(StrAndUnicode):
     "A Field plus data"
     def __init__(self, form, field, name):

django/forms/models.py

@@ -12 +12 @@
 from django.core.exceptions import ValidationError, NON_FIELD_ERRORS
 from django.core.validators import EMPTY_VALUES
 from util import ErrorList
-from forms import BaseForm, get_declared_fields
+from forms import BaseForm, BaseRequestFormTraits, get_declared_fields
 from fields import Field, ChoiceField
 from widgets import SelectMultiple, HiddenInput, MultipleHiddenInput
 from widgets import media_property
@@ -21 +21 @@
 __all__ = (
     'ModelForm', 'BaseModelForm', 'model_to_dict', 'fields_for_model',
     'save_instance', 'form_for_fields', 'ModelChoiceField',
-    'ModelMultipleChoiceField',
+    'ModelMultipleChoiceField', 'RequestModelForm',
 )
 
 def construct_instance(form, instance, fields=None, exclude=None):
@@ -201 +201 @@
         formfield_callback = attrs.pop('formfield_callback',
                 lambda f, **kwargs: f.formfield(**kwargs))
         try:
-            parents = [b for b in bases if issubclass(b, ModelForm)]
+            parents = [b for b in bases if issubclass(b, (BaseModelForm))]
         except NameError:
             # We are defining ModelForm itself.
             parents = None
@@ -375 +375 @@
 class ModelForm(BaseModelForm):
     __metaclass__ = ModelFormMetaclass
 
+class RequestModelForm(BaseRequestFormTraits, BaseModelForm):
+    __metaclass__ = ModelFormMetaclass
+
 def modelform_factory(model, form=ModelForm, fields=None, exclude=None,
                        formfield_callback=lambda f: f.formfield()):
     # Create the inner Meta class. FIXME: ideally, we should be able to

django/http/multipartparser.py

@@ -169 +169 @@
                     file_name = self.IE_sanitize(unescape_entities(file_name))
 
                     content_type = meta_data.get('content-type', ('',))[0].strip()
+                    content_type_extra = meta_data.get('content-type', (0,{}))[1]
                     try:
-                        charset = meta_data.get('content-type', (0,{}))[1].get('charset', None)
+                        charset = content_type_extra.get('charset', None)
                     except:
                         charset = None
 
@@ -185 +186 @@
                             try:
                                 handler.new_file(field_name, file_name,
                                                  content_type, content_length,
-                                                 charset)
+                                                 charset, content_type_extra)
                             except StopFutureHandlers:
                                 break
 

new file tests/regressiontests/file_uploads/backends.py

@@ +1 @@
+from django.conf import settings
+from django.core.files.transfers import BaseFileBackend
+from django.http import HttpResponseRedirect
+import models
+
+class TestFileBackend(BaseFileBackend):
+    def prepare_upload(self, request, url, *args, **kwargs):
+        return '/redirect' + url, {'x-extra': 'abcd', 'x-data': 'efg'}
+
+    def serve(self, request, file, *args, **kwargs):
+        if not file.name.endswith('pass'):
+            return HttpResponseRedirect('/download/')
+
+    def download_url(self, file):
+        if not file.name.endswith('pass'):
+            return '/public/'
+
+    def get_storage_backend(self, *args, **kwargs):
+        return models.temp_storage

new file tests/regressiontests/file_uploads/forms.py

@@ +1 @@
+from django import forms
+from models import FileModel
+
+class FileForm(forms.RequestModelForm):
+    class Meta:
+        model = FileModel

tests/regressiontests/file_uploads/models.py

@@ -1 +1 @@
 import tempfile
 import os
+from django.conf import settings
 from django.db import models
 from django.core.files.storage import FileSystemStorage
 
 temp_storage = FileSystemStorage(tempfile.mkdtemp())
 UPLOAD_TO = os.path.join(temp_storage.location, 'test_upload')
+settings.DEFAULT_FILE_TRANSFER_BACKENDS.update({
+    'regressiontests.file_uploads.*': 'regressiontests.file_uploads.backends.TestFileBackend',
+})
 
 class FileModel(models.Model):
-    testfile = models.FileField(storage=temp_storage, upload_to='test_upload')
+    testfile = models.FileField(upload_to='test_upload')

tests/regressiontests/file_uploads/tests.py

@@ -5 +5 @@
 import unittest
 from StringIO import StringIO
 
+from django.conf import settings
 from django.core.files import temp as tempfile
 from django.core.files.uploadedfile import SimpleUploadedFile
 from django.test import TestCase, client
@@ -17 +18 @@
 
 UNICODE_FILENAME = u'test-0123456789_中文_Orléans.jpg'
 
+class FileBackendTests(TestCase):
+    def setUp(self):
+        if not os.path.isdir(temp_storage.location):
+            os.makedirs(temp_storage.location)
+        if os.path.isdir(UPLOAD_TO):
+            shutil.rmtree(UPLOAD_TO)
+
+    def tearDown(self):
+        shutil.rmtree(temp_storage.location)
+
+    def test_prepare_upload(self):
+        response = self.client.get('/file_uploads/prepare_upload/')
+        self.assertEqual(response.status_code, 200)
+        data = simplejson.loads(response.content)
+        self.assertEqual(data, [
+            '/redirect/file_uploads/prepare_upload/', [
+                '<input type="file" name="testfile" id="id_testfile" />',
+                '<input type="hidden" name="x-data" value="efg" id="id_x-data" />',
+                '<input type="hidden" name="x-extra" value="abcd" id="id_x-extra" />',
+            ]
+        ])
+
+    def test_serve_file(self):
+        obj = FileModel()
+        obj.testfile.save('foo.txt', SimpleUploadedFile('foo.txt', 'x'))
+        obj.save()
+        response = self.client.get('/file_uploads/serve/')
+        self.assertEqual(response.status_code, 302)
+        self.assertEqual(response['Location'], 'http://testserver/download/')
+
+    def test_serve_file_fallback(self):
+        obj = FileModel()
+        obj.testfile.save('foo.pass', SimpleUploadedFile('foo.pass', 'x'))
+        obj.save()
+        response = self.client.get('/file_uploads/serve/')
+        self.assertEqual(response.status_code, 200)
+
+    def test_download_url(self):
+        obj = FileModel()
+        obj.testfile.save('foo2.txt', SimpleUploadedFile('foo2.txt', 'x'))
+        obj.save()
+        url = obj.testfile.download_url()
+        self.assertEqual(url, '/public/')
+
+    def test_download_url_fallback(self):
+        obj = FileModel()
+        obj.testfile.save('foo2.pass', SimpleUploadedFile('foo2.pass', 'x'))
+        obj.save()
+        url = obj.testfile.download_url()
+        self.assertEqual(url, settings.MEDIA_URL + obj.testfile.name)
+
 class FileUploadTests(TestCase):
     def test_simple_upload(self):
         post_data = {
@@ -122 +174 @@
         response = self.client.request(**r)
 
         # The filenames should have been sanitized by the time it got to the view.
-        recieved = simplejson.loads(response.content)
+        received = simplejson.loads(response.content)
         for i, name in enumerate(scary_file_names):
-            got = recieved["file%s" % i]
+            got = received["file%s" % i]
             self.assertEqual(got, "hax0rd.txt")
 
     def test_filename_overflow(self):

tests/regressiontests/file_uploads/urls.py

@@ -10 +10 @@
     (r'^quota/broken/$',    views.file_upload_quota_broken),
     (r'^getlist_count/$',   views.file_upload_getlist_count),
     (r'^upload_errors/$',   views.file_upload_errors),
+    (r'^prepare_upload/$',  views.file_prepare_upload),
+    (r'^serve/$', views.file_serve),
 )

tests/regressiontests/file_uploads/views.py

@@ -2 +2 @@
 from django.core.files.uploadedfile import UploadedFile
 from django.http import HttpResponse, HttpResponseServerError
 from django.utils import simplejson
+from forms import FileForm
 from models import FileModel, UPLOAD_TO
 from uploadhandler import QuotaUploadHandler, ErroringUploadHandler
 from django.utils.hashcompat import sha_constructor
@@ -112 +113 @@
 def file_upload_errors(request):
     request.upload_handlers.insert(0, ErroringUploadHandler())
     return file_upload_echo(request)
+
+def file_prepare_upload(request):
+    form = FileForm(request)
+    return HttpResponse(simplejson.dumps([form.upload_url, list(map(unicode, form))]))
+
+def file_serve(request):
+    return FileModel.objects.all()[0].testfile.serve(request)