Changeset 7814

Timestamp:

07/01/08 10:10:51 (2 months ago)

Author:

jacob

Message:

Fixed #2070: refactored Django's file upload capabilities.

A description of the new features can be found in the new upload handling documentation; the executive summary is that Django will now happily handle uploads of large files without issues.

This changes the representation of uploaded files from dictionaries to bona fide objects; see BackwardsIncompatibleChanges for details.

Files:

• django/trunk/AUTHORS (modified) (2 diffs)
• django/trunk/django/conf/global_settings.py (modified) (1 diff)
• django/trunk/django/core/files (added)
• django/trunk/django/core/files/__init__.py (added)
• django/trunk/django/core/files/locks.py (added)
• django/trunk/django/core/files/move.py (added)
• django/trunk/django/core/files/uploadedfile.py (added)
• django/trunk/django/core/files/uploadhandler.py (added)
• django/trunk/django/core/handlers/modpython.py (modified) (1 diff)
• django/trunk/django/core/handlers/wsgi.py (modified) (1 diff)
• django/trunk/django/db/models/base.py (modified) (4 diffs)
• django/trunk/django/db/models/fields/__init__.py (modified) (3 diffs)
• django/trunk/django/http/__init__.py (modified) (3 diffs)
• django/trunk/django/http/multipartparser.py (added)
• django/trunk/django/newforms/fields.py (modified) (4 diffs)
• django/trunk/django/oldforms/__init__.py (modified) (2 diffs)
• django/trunk/django/test/client.py (modified) (3 diffs)
• django/trunk/django/utils/datastructures.py (modified) (1 diff)
• django/trunk/django/utils/text.py (modified) (2 diffs)
• django/trunk/docs/newforms.txt (modified) (1 diff)
• django/trunk/docs/request_response.txt (modified) (1 diff)
• django/trunk/docs/settings.txt (modified) (2 diffs)
• django/trunk/docs/upload_handling.txt (added)
• django/trunk/tests/modeltests/model_forms/models.py (modified) (13 diffs)
• django/trunk/tests/regressiontests/bug639/tests.py (modified) (2 diffs)
• django/trunk/tests/regressiontests/datastructures/tests.py (modified) (1 diff)
• django/trunk/tests/regressiontests/file_uploads (added)
• django/trunk/tests/regressiontests/file_uploads/__init__.py (added)
• django/trunk/tests/regressiontests/file_uploads/models.py (added)
• django/trunk/tests/regressiontests/file_uploads/tests.py (added)
• django/trunk/tests/regressiontests/file_uploads/uploadhandler.py (added)
• django/trunk/tests/regressiontests/file_uploads/urls.py (added)
• django/trunk/tests/regressiontests/file_uploads/views.py (added)
• django/trunk/tests/regressiontests/forms/error_messages.py (modified) (2 diffs)
• django/trunk/tests/regressiontests/forms/fields.py (modified) (3 diffs)
• django/trunk/tests/regressiontests/forms/forms.py (modified) (3 diffs)
• django/trunk/tests/regressiontests/test_client_regress/models.py (modified) (3 diffs)
• django/trunk/tests/regressiontests/test_client_regress/urls.py (modified) (1 diff)
• django/trunk/tests/regressiontests/test_client_regress/views.py (modified) (3 diffs)
• django/trunk/tests/urls.py (modified) (1 diff)

django/trunk/AUTHORS

@@ -60 +60 @@
-    av0000@mail.ru
-    David Avsajanishvili <avsd05@gmail.com>
-axiak@mit.edu
+Mike Axiak <axiak@mit.edu>
-    Niran Babalola <niran@niran.org>
-    Morten Bagai <m@bagai.com>
@@ -142 +142 @@
-    Szilveszter Farkas <szilveszter.farkas@gmail.com>
-    favo@exoweb.net
+    fdr <drfarina@gmail.com>
-    Dmitri Fedortchenko <zeraien@gmail.com>
+    Jonathan Feignberg <jdf@pobox.com>
-    Liang Feng <hutuworm@gmail.com>
-    Bill Fenner <fenner@gmail.com>

django/trunk/django/conf/global_settings.py

@@ -232 +232 @@
-MEDIA_URL = ''
-
+# List of upload handler classes to be applied in order.
+FILE_UPLOAD_HANDLERS = (
+    'django.core.files.uploadhandler.MemoryFileUploadHandler',
+    'django.core.files.uploadhandler.TemporaryFileUploadHandler',
+)
+
+# Maximum size, in bytes, of a request before it will be streamed to the
+# file system instead of into memory.
+FILE_UPLOAD_MAX_MEMORY_SIZE = 2621440 # i.e. 2.5 MB
+
+# Directory in which upload streamed files will be temporarily saved. A value of
+# `None` will make Django use the operating system's default temporary directory
+# (i.e. "/tmp" on *nix systems).
+FILE_UPLOAD_TEMP_DIR = None
+
-# Default formatting for date objects. See all available format strings here:
-# http://www.djangoproject.com/documentation/templates/#now

django/trunk/django/core/handlers/modpython.py

@@ -54 +54 @@
-        "Populates self._post and self._files"
-        if 'content-type' in self._req.headers_in and self._req.headers_in['content-type'].startswith('multipart'):
-
+
+
-        else:
-            self._post, self._files = http.QueryDict(self.raw_post_data, encoding=self._encoding), datastructures.MultiValueDict()

django/trunk/django/core/handlers/wsgi.py

@@ -113 +113 @@
-        if self.method == 'POST':
-            if self.environ.get('CONTENT_TYPE', '').startswith('multipart'):
-
-
-
+
+
-            else:
-                self._post, self._files = http.QueryDict(self.raw_post_data, encoding=self._encoding), datastructures.MultiValueDict()

django/trunk/django/db/models/base.py

@@ -20 +20 @@
-from django.utils.functional import curry
-from django.utils.encoding import smart_str, force_unicode, smart_unicode
+from django.core.files.move import file_move_safe
+from django.core.files import locks
-from django.conf import settings
-
@@ -470 +472 @@
-        return os.path.getsize(self._get_FIELD_filename(field))
-
-contents
+field
-        directory = field.get_directory_name()
-        try: # Create the date-based directory if it doesn't exist.
@@ -476 +478 @@
-        except OSError: # Directory probably already exists.
-            pass
+
+        #
+        # Check for old-style usage (files-as-dictionaries). Warn here first
+        # since there are multiple locations where we need to support both new
+        # and old usage.
+        #
+        if isinstance(raw_field, dict):
+            import warnings
+            warnings.warn(
+                message = "Representing uploaded files as dictionaries is"\
+                          " deprected. Use django.core.files.SimpleUploadedFile"\
+                          " instead.",
+                category = DeprecationWarning,
+                stacklevel = 2
+            )
+            from django.core.files.uploadedfile import SimpleUploadedFile
+            raw_field = SimpleUploadedFile.from_dict(raw_field)
+
+        elif isinstance(raw_field, basestring):
+            import warnings
+            warnings.warn(
+                message = "Representing uploaded files as strings is "\
+                          " deprecated. Use django.core.files.SimpleUploadedFile "\
+                          " instead.",
+                category = DeprecationWarning,
+                stacklevel = 2
+            )
+            from django.core.files.uploadedfile import SimpleUploadedFile
+            raw_field = SimpleUploadedFile(filename, raw_field)
+
+        if filename is None:
+            filename = raw_field.file_name
+
-        filename = field.get_filename(filename)
-
+        #
-        # If the filename already exists, keep adding an underscore to the name of
-        # the file until the filename doesn't exist.
+        #
-        while os.path.exists(os.path.join(settings.MEDIA_ROOT, filename)):
-            try:
@@ -487 +524 @@
-            else:
-                filename = filename[:dot_index] + '_' + filename[dot_index:]
-
-
+
+
+
+
-        setattr(self, field.attname, filename)
-
-        full_filename = self._get_FIELD_filename(field)
-
-
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
-
-        # Save the width and/or height, if applicable.

django/trunk/django/db/models/fields/__init__.py

@@ -812 +812 @@
-        setattr(cls, 'get_%s_url' % self.name, curry(cls._get_FIELD_url, field=self))
-        setattr(cls, 'get_%s_size' % self.name, curry(cls._get_FIELD_size, field=self))
-contents, save=True: instance._save_FIELD_file(self, filename, raw_contents
+field, save=True: instance._save_FIELD_file(self, filename, raw_field
-        dispatcher.connect(self.delete_file, signal=signals.post_delete, sender=cls)
-
@@ -835 +835 @@
-            func = getattr(new_object, 'save_%s_file' % self.name)
-            if rel:
-unc(new_data[upload_field_name][0]["filename"], new_data[upload_field_name][0]["content"], save)
+ile = new_data[upload_field_name][0]
-            else:
-
+
+
+
+
+
+
+
+
+
+
+
-
-    def get_directory_name(self):
@@ -850 +860 @@
-        from django.newforms.fields import UploadedFile
-        if data and isinstance(data, UploadedFile):
-content
+data
-
-    def formfield(self, **kwargs):

django/trunk/django/http/__init__.py

@@ -10 +10 @@
-    from cgi import parse_qsl
-
-FileDic
+ImmutableLis
-from django.utils.encoding import smart_str, iri_to_uri, force_unicode
-
+
+
+
-from utils import *
-
-RESERVED_CHARS="!*'();:@&=+$,/?%#[]"
-
-
-class Http404(Exception):
@@ -26 +27 @@
-    # The encoding used in GET/POST dicts. None means use default setting.
-    _encoding = None
+    _upload_handlers = []
-
-    def __init__(self):
@@ -103 +105 @@
-    encoding = property(_get_encoding, _set_encoding)
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
-
-class QueryDict(MultiValueDict):

django/trunk/django/newforms/fields.py

@@ -8 +8 @@
-import re
-import time
+try:
+    from cStringIO import StringIO
+except ImportError:
+    from StringIO import StringIO
+
-# Python 2.3 fallbacks
-try:
@@ -417 +422 @@
-class UploadedFile(StrAndUnicode):
-    "A wrapper for files uploaded in a FileField"
-content
+data
-        self.filename = filename
-content = content
+data = data
-
-    def __unicode__(self):
@@ -445 +450 @@
-        elif not data and initial:
-            return initial
+
+        if isinstance(data, dict):
+            # We warn once, then support both ways below.
+            import warnings
+            warnings.warn(
+                message = "Representing uploaded files as dictionaries is"\
+                          " deprecated. Use django.core.files.SimpleUploadedFile "\
+                          " instead.",
+                category = DeprecationWarning,
+                stacklevel = 2
+            )
+
-        try:
-
-
+
+
+
+
+
+
+
+
+
+
-            raise ValidationError(self.error_messages['invalid'])
-
-
-
+
-            raise ValidationError(self.error_messages['empty'])
-
+
+
-
-class ImageField(FileField):
@@ -471 +495 @@
-            return initial
-        from PIL import Image
-
+
+
+
+
+
+
+
+
+
+
+
-        try:
-            # load() is the only method that can spot a truncated JPEG,
-            #  but it cannot be called sanely after verify()
-StringIO(f.content)
+file
-            trial_image.load()
+
+            # Since we're about to use the file again we have to reset the
+            # file object if possible.
+            if hasattr(file, 'reset'):
+                file.reset()
+
-            # verify() is the only method that can spot a corrupt PNG,
-            #  but it must be called immediately after the constructor
-StringIO(f.content)
+file
-            trial_image.verify()
-        except Exception: # Python Imaging Library doesn't recognize it as an image

django/trunk/django/oldforms/__init__.py

@@ -681 +681 @@
-        self.validator_list = [self.isNonEmptyFile] + validator_list
-
-
-
-
-
-
-
+
+
+
+
+
+
+
+
+
+
-            raise validators.CriticalValidationError, ugettext("The submitted file is empty.")
-
@@ -692 +696 @@
-        return mark_safe(u'<input type="file" id="%s" class="v%s" name="%s" />' % \
-            (self.get_id(), self.__class__.__name__, self.field_name))
+
+    def prepare(self, new_data):
+        if hasattr(new_data, 'upload_errors'):
+            upload_errors = new_data.upload_errors()
+            new_data[self.field_name] = { '_file_upload_error': upload_errors }
-
-    def html2python(data):

django/trunk/django/test/client.py

@@ -2 +2 @@
-import sys
-import os
-
+
+
+
+
-from django.conf import settings
-from django.contrib.auth import authenticate, login
@@ -19 +22 @@
-BOUNDARY = 'BoUnDaRyStRiNg'
-MULTIPART_CONTENT = 'multipart/form-data; boundary=%s' % BOUNDARY
+
+class FakePayload(object):
+    """
+    A wrapper around StringIO that restricts what can be read since data from
+    the network can't be seeked and cannot be read outside of its content
+    length. This makes sure that views can't do anything under the test client
+    that wouldn't work in Real Life.
+    """
+    def __init__(self, content):
+        self.__content = StringIO(content)
+        self.__len = len(content)
+
+    def read(self, num_bytes=None):
+        if num_bytes is None:
+            num_bytes = self.__len or 1
+        assert self.__len >= num_bytes, "Cannot read more than the available bytes from the HTTP incoming data."
+        content = self.__content.read(num_bytes)
+        self.__len -= num_bytes
+        return content
-
-class ClientHandler(BaseHandler):
@@ -237 +259 @@
-            'PATH_INFO':      urllib.unquote(path),
-            'REQUEST_METHOD': 'POST',
-StringIO
+FakePayload
-        }
-        r.update(extra)

django/trunk/django/utils/datastructures.py

@@ -333 +333 @@
-                current = {bits[-1]: v}
-
-
-
-
-
-
-
-
-
-
-
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
-
-class DictWrapper(dict):

django/trunk/django/utils/text.py

@@ -4 +4 @@
-from django.utils.functional import allow_lazy
-from django.utils.translation import ugettext_lazy
+from htmlentitydefs import name2codepoint
-
-# Capitalizes the first letter of a string.
@@ -223 +224 @@
-smart_split = allow_lazy(smart_split, unicode)
-
+def _replace_entity(match):
+     text = match.group(1)
+     if text[0] == u'#':
+         text = text[1:]
+         try:
+             if text[0] in u'xX':
+                 c = int(text[1:], 16)
+             else:
+                 c = int(text)
+             return unichr(c)
+         except ValueError:
+             return match.group(0)
+     else:
+         try:
+             return unichr(name2codepoint[text])
+         except (ValueError, KeyError):
+             return match.group(0)
+
+_entity_re = re.compile(r"&(#?[xX]?(?:[0-9a-fA-F]+|\w{1,8}));")
+
+def unescape_entities(text):
+     return _entity_re.sub(_replace_entity, text)
+unescape_entities = allow_lazy(unescape_entities, unicode)

django/trunk/docs/newforms.txt

@@ -806 +806 @@
-
-    # Bound form with an image field
+    >>> from django.core.files.uploadedfile import SimpleUploadedFile
-    >>> data = {'subject': 'hello',
-    ...         'message': 'Hi there',
-    ...         'sender': 'foo@example.com',
-    ...         'cc_myself': True}
-
-
+
-    >>> f = ContactFormWithMugshot(data, file_data)
-

django/trunk/docs/request_response.txt

@@ -81 +81 @@
-
-``FILES``
+    
+    .. admonition:: Changed in Django development version
+        
+        In previous versions of Django, ``request.FILES`` contained
+        simple ``dict`` objects representing uploaded files. This is
+        no longer true -- files are represented by ``UploadedFile``
+        objects as described below.
+        
+        These ``UploadedFile`` objects will emulate the old-style ``dict``
+        interface, but this is deprecated and will be removed in the next
+        release of Django.
+        
-    A dictionary-like object containing all uploaded files. Each key in
-    ``FILES`` is the ``name`` from the ``<input type="file" name="" />``. Each
-
-
-
-
-
-
-
+
+
+
+
+
+
+
+
+
+
-    Note that ``FILES`` will only contain data if the request method was POST
-    and the ``<form>`` that posted to the request had
-    ``enctype="multipart/form-data"``. Otherwise, ``FILES`` will be a blank
-    dictionary-like object.
+    
+    .. _File Uploads: ../upload_handling/
-
-``META``

django/trunk/docs/settings.txt

@@ -280 +280 @@
-The database backend to use. The build-in database backends are
-``'postgresql_psycopg2'``, ``'postgresql'``, ``'mysql'``, ``'mysql_old'``,
-
+, ``'oracle'``,
-
-In the Django development version, you can use a database backend that doesn't
@@ -530 +530 @@
-The character encoding used to decode any files read from disk. This includes
-template files and initial SQL data files.
+
+FILE_UPLOAD_HANDLERS
+--------------------
+
+**New in Django development version**
+
+Default::
+
+    ("django.core.files.fileuploadhandler.MemoryFileUploadHandler",
+     "django.core.files.fileuploadhandler.TemporaryFileUploadHandler",)
+
+A tuple of handlers to use for uploading. See `file uploads`_ for details.
+
+.. _file uploads: ../upload_handling/
+
+FILE_UPLOAD_MAX_MEMORY_SIZE
+---------------------------
+
+**New in Django development version**
+
+Default: ``2621440`` (i.e. 2.5 MB).
+
+The maximum size (in bytes) that an upload will be before it gets streamed to
+the file system. See `file uploads`_ for details.
+
+FILE_UPLOAD_TEMP_DIR
+--------------------
+
+**New in Django development version**
+
+Default: ``None``
+
+The directory to store data temporarily while uploading files. If ``None``,
+Django will use the standard temporary directory for the operating system. For
+example, this will default to '/tmp' on *nix-style operating systems.
+
+See `file uploads`_ for details.
-
-FIXTURE_DIRS

django/trunk/tests/modeltests/model_forms/models.py

@@ -68 +68 @@
-class ImageFile(models.Model):
-    description = models.CharField(max_length=20)
-
+
+
+
+
+
+
+
-
-    def __unicode__(self):
@@ -76 +82 @@
->>> from django import newforms as forms
->>> from django.newforms.models import ModelForm
+>>> from django.core.files.uploadedfile import SimpleUploadedFile
-
-The bare bones, absolutely nothing custom, basic case.
@@ -793 +800 @@
-# Upload a file and ensure it all works as expected.
-
-{'filename': 'test1.txt', 'content': 'hello world'}
+SimpleUploadedFile('test1.txt', 'hello world')
->>> f.is_valid()
-True
@@ -802 +809 @@
-u'...test1.txt'
-
+>>> os.unlink(instance.get_file_filename())
+
+>>> f = TextFileForm(data={'description': u'Assistance'}, files={'file': SimpleUploadedFile('test1.txt', 'hello world')})
+>>> f.is_valid()
+True
+>>> type(f.cleaned_data['file'])
+<class 'django.newforms.fields.UploadedFile'>
+>>> instance = f.save()
+>>> instance.file
+u'...test1.txt'
+
-# Edit an instance that already has the file defined in the model. This will not
-# save the file again, but leave it exactly as it is.
@@ -815 +833 @@
-
-# Delete the current file since this is not done by Django.
-
->>> os.unlink(instance.get_file_filename())
-
-# Override the file by uploading a new one.
-
-{'filename': 'test2.txt', 'content': 'hello world'}
+SimpleUploadedFile('test2.txt', 'hello world')
->>> f.is_valid()
-True
@@ -827 +844 @@
-u'...test2.txt'
-
+# Delete the current file since this is not done by Django.
+>>> os.unlink(instance.get_file_filename())
+
+>>> f = TextFileForm(data={'description': u'Assistance'}, files={'file': SimpleUploadedFile('test2.txt', 'hello world')})
+>>> f.is_valid()
+True
+>>> instance = f.save()
+>>> instance.file
+u'...test2.txt'
+
+# Delete the current file since this is not done by Django.
+>>> os.unlink(instance.get_file_filename())
+
->>> instance.delete()
-
@@ -839 +869 @@
-''
-
-{'filename': 'test3.txt', 'content': 'hello world'}
+SimpleUploadedFile('test3.txt', 'hello world')
->>> f.is_valid()
-True
@@ -845 +875 @@
->>> instance.file
-u'...test3.txt'
+
+# Delete the current file since this is not done by Django.
+>>> os.unlink(instance.get_file_filename())
+>>> instance.delete()
+
+>>> f = TextFileForm(data={'description': u'Assistance'}, files={'file': SimpleUploadedFile('test3.txt', 'hello world')})
+>>> f.is_valid()
+True
+>>> instance = f.save()
+>>> instance.file
+u'...test3.txt'
+
+# Delete the current file since this is not done by Django.
+>>> os.unlink(instance.get_file_filename())
->>> instance.delete()
-
@@ -859 +903 @@
->>> image_data = open(os.path.join(os.path.dirname(__file__), "test.png")).read()
-
-{'filename': 'test.png', 'content': image_data}
+SimpleUploadedFile('test.png', image_data)
->>> f.is_valid()
-True
@@ -868 +912 @@
-u'...test.png'
-
+# Delete the current file since this is not done by Django.
+>>> os.unlink(instance.get_image_filename())
+
+>>> f = ImageFileForm(data={'description': u'An image'}, files={'image': SimpleUploadedFile('test.png', image_data)})
+>>> f.is_valid()
+True
+>>> type(f.cleaned_data['image'])
+<class 'django.newforms.fields.UploadedFile'>
+>>> instance = f.save()
+>>> instance.image
+u'...test.png'
+
-# Edit an instance that already has the image defined in the model. This will not
-# save the image again, but leave it exactly as it is.
@@ -886 +942 @@
-# Override the file by uploading a new one.
-
-{'filename': 'test2.png', 'content': image_data}
+SimpleUploadedFile('test2.png', image_data)
->>> f.is_valid()
-True
@@ -893 +949 @@
-u'...test2.png'
-
+# Delete the current file since this is not done by Django.
+>>> os.unlink(instance.get_image_filename())
+>>> instance.delete()
+
+>>> f = ImageFileForm(data={'description': u'Changed it'}, files={'image': SimpleUploadedFile('test2.png', image_data)})
+>>> f.is_valid()
+True
+>>> instance = f.save()
+>>> instance.image
+u'...test2.png'
+
+# Delete the current file since this is not done by Django.
+>>> os.unlink(instance.get_image_filename())
->>> instance.delete()
-
@@ -905 +974 @@
-''
-
-
+
+