Ticket #4165: 5099-streaming_file_upload_with_safe_file_move_progress.diff

django/http/__init__.py

@@ -1 @@
-import os
+import os, pickle
 from Cookie import SimpleCookie
 from pprint import pformat
 from urllib import urlencode, quote
 from django.utils.datastructures import MultiValueDict
 
+try:
+    from cStringIO import StringIO
+except ImportError:
+    from StringIO import StringIO
+
 RESERVED_CHARS="!*'();:@&=+$,/?%#[]"
 
 try:
@@ -306 +311 @@
     if not host:
         host = request.META.get('HTTP_HOST', '')
     return host
+

django/oldforms/__init__.py

@@ -666 +666 @@
         self.validator_list = [self.isNonEmptyFile] + validator_list
 
     def isNonEmptyFile(self, field_data, all_data):
-        try:
-            content = field_data['content']
-        except TypeError:
+        if field_data.has_key('_file_upload_error'):
+            raise validators.CriticalValidationError, field_data['_file_upload_error']
+        if not field_data.has_key('filename'):
             raise validators.CriticalValidationError, gettext("No file was submitted. Check the encoding type on the form.")
-        if not content:
+        if not field_data['content-length']:
             raise validators.CriticalValidationError, gettext("The submitted file is empty.")
 
     def render(self, data):
         return '<input type="file" id="%s" class="v%s" name="%s" />' % \
             (self.get_id(), self.__class__.__name__, self.field_name)
 
+    def prepare(self, new_data):
+        if new_data.has_key('_file_upload_error'):
+            # pretend we got something in the field to raise a validation error later
+            new_data[self.field_name] = { '_file_upload_error': new_data['_file_upload_error'] }
+
     def html2python(data):
         if data is None:
             raise EmptyValue

django/db/models/base.py

@@ -12 +12 @@
 from django.dispatch import dispatcher
 from django.utils.datastructures import SortedDict
 from django.utils.functional import curry
+from django.utils.file import file_move_safe
 from django.conf import settings
 from itertools import izip
 import types
 import sys
 import os
 
+                
 class ModelBase(type):
     "Metaclass for all models"
     def __new__(cls, name, bases, attrs):
@@ -361 +363 @@
     def _get_FIELD_size(self, field):
         return os.path.getsize(self._get_FIELD_filename(field))
 
-    def _save_FIELD_file(self, field, filename, raw_contents, save=True):
+    def _save_FIELD_file(self, field, filename, raw_field, save=True):
         directory = field.get_directory_name()
         try: # Create the date-based directory if it doesn't exist.
             os.makedirs(os.path.join(settings.MEDIA_ROOT, directory))
@@ -383 +385 @@
         setattr(self, field.attname, filename)
 
         full_filename = self._get_FIELD_filename(field)
-        fp = open(full_filename, 'wb')
-        fp.write(raw_contents)
-        fp.close()
+        if raw_field.has_key('tmpfilename'):
+            raw_field['tmpfile'].close()
+            file_move_safe(raw_field['tmpfilename'], full_filename)
+        else:
+            fp = open(full_filename, 'wb')
+            fp.write(raw_field['content'])
+            fp.close()
 
         # Save the width and/or height, if applicable.
         if isinstance(field, ImageField) and (field.width_field or field.height_field):

django/db/models/fields/__init__.py

@@ -636 +636 @@
         setattr(cls, 'get_%s_filename' % self.name, curry(cls._get_FIELD_filename, field=self))
         setattr(cls, 'get_%s_url' % self.name, curry(cls._get_FIELD_url, field=self))
         setattr(cls, 'get_%s_size' % self.name, curry(cls._get_FIELD_size, field=self))
-        setattr(cls, 'save_%s_file' % self.name, lambda instance, filename, raw_contents, save=True: instance._save_FIELD_file(self, filename, raw_contents, save))
+        setattr(cls, 'save_%s_file' % self.name, lambda instance, filename, raw_field, save=True: instance._save_FIELD_file(self, filename, raw_field, save))
         dispatcher.connect(self.delete_file, signal=signals.post_delete, sender=cls)
 
     def delete_file(self, instance):
@@ -659 +659 @@
         if new_data.get(upload_field_name, False):
             func = getattr(new_object, 'save_%s_file' % self.name)
             if rel:
-                func(new_data[upload_field_name][0]["filename"], new_data[upload_field_name][0]["content"], save)
+                func(new_data[upload_field_name][0]["filename"], new_data[upload_field_name][0], save)
             else:
-                func(new_data[upload_field_name]["filename"], new_data[upload_field_name]["content"], save)
+                func(new_data[upload_field_name]["filename"], new_data[upload_field_name], save)
 
     def get_directory_name(self):
         return os.path.normpath(datetime.datetime.now().strftime(self.upload_to))

django/conf/global_settings.py

@@ -240 +240 @@
 # isExistingURL validator.
 URL_VALIDATOR_USER_AGENT = "Django/0.96pre (http://www.djangoproject.com)"
 
+# The directory to place streamed file uploads. The web server needs write
+# permissions on this directory.
+# If this is None, streaming uploads are disabled.
+FILE_UPLOAD_DIR = None
+
+
+# The minimum size of a POST before file uploads are streamed to disk.
+# Any less than this number, and the file is uploaded to memory.
+# Size is in bytes.
+STREAMING_MIN_POST_SIZE = 512 * (2**10)
+
+
+
+
 ##############
 # MIDDLEWARE #
 ##############
@@ -335 +349 @@
 
 # The list of directories to search for fixtures
 FIXTURE_DIRS = ()
+
+

django/core/handlers/wsgi.py

@@ -111 +111 @@
             if self.environ.get('CONTENT_TYPE', '').startswith('multipart'):
                 header_dict = dict([(k, v) for k, v in self.environ.items() if k.startswith('HTTP_')])
                 header_dict['Content-Type'] = self.environ.get('CONTENT_TYPE', '')
-                self._post, self._files = http.parse_file_upload(header_dict, self.raw_post_data)
+                header_dict['Content-Length'] = self.environ.get('CONTENT_LENGTH', '')
+                try:
+                    self._post, self._files = http.parse_file_upload(header_dict, self.environ['wsgi.input'], self.META.get('QUERY_STRING'))
+                except:
+                    self._post, self._files = {}, {} # make sure we dont read the input stream again
+                    raise
+                self._raw_post_data = None # raw data is not available for streamed multipart messages
             else:
                 self._post, self._files = http.QueryDict(self.raw_post_data), datastructures.MultiValueDict()
         else:

django/core/handlers/modpython.py

@@ -47 +47 @@
     def _load_post_and_files(self):
         "Populates self._post and self._files"
         if 'content-type' in self._req.headers_in and self._req.headers_in['content-type'].startswith('multipart'):
-            self._post, self._files = http.parse_file_upload(self._req.headers_in, self.raw_post_data)
+            self._raw_post_data = None # raw data is not available for streamed multipart messages
+            try:
+                self._post, self._files = http.parse_file_upload(self._req.headers_in, self._req, self.META.get('QUERY_STRING'))
+            except:
+                self._post, self._files = {}, {} # make sure we dont read the input stream again
+                raise
         else:
             self._post, self._files = http.QueryDict(self.raw_post_data), datastructures.MultiValueDict()
 

django/utils/file.py

@@ +1 @@
+import os
+
+try:
+    import shutils
+    file_move = shutils.move
+except:
+    file_move = os.rename
+
+def file_move_safe(old_file_name, new_file_name, chunk_size = 1024*64):
+    """
+    Moves a file from one location to another in the safest way possible.
+    
+    First, it tries using shutils.move, which is OS-dependent but doesn't
+    break with change of filesystems. Then it tries os.rename, which will
+    break if it encounters a change in filesystems. Lastly, it streams
+    it manually from one file to another in python.
+    """
+    
+    try:
+        file_move(old_file_name, new_file_name)
+        return
+    except:
+        pass
+    
+    new_file = open(new_file_name, 'wb')
+    old_file = open(old_file_name, 'rb')
+    current_chunk = None
+    
+    while current_chunk != '':
+        current_chunk = old_file.read(chunk_size)
+        new_file.write(current_chunk)
+        
+    new_file.close()
+    old_file.close()
+
+    os.remove(old_file_name)

tests/modeltests/test_client/views.py

@@ -44 +44 @@
 
     return HttpResponse(t.render(c))
 
+def post_file_view(request):
+    "A view that expects a multipart post and returns a file in the context"
+    t = Template('File {{ file.filename }} received', name='POST Template')
+    c = Context({'file': request.FILES['file_file']})
+    return HttpResponse(t.render(c))
+
 def redirect_view(request):
     "A view that redirects all requests to the GET view"
     return HttpResponseRedirect('/test_client/get_view/')

tests/modeltests/test_client/models.py

@@ -75 +75 @@
         self.assertEqual(response.template.name, "Book template")
         self.assertEqual(response.content, "Blink - Malcolm Gladwell")
 
+    def test_post_file_view(self):
+        "POST this python file to a view"
+        import os, tempfile
+        from django.conf import settings
+        file = __file__.replace('.pyc', '.py')
+        for upload_dir in [None, tempfile.gettempdir()]:
+            settings.FILE_UPLOAD_DIR = upload_dir
+            post_data = { 'name': file, 'file': open(file) }
+            response = self.client.post('/test_client/post_file_view/', post_data)
+            self.failUnless('models.py' in response.context['file']['filename'])
+            self.failUnless(len(response.context['file']['content']) == os.path.getsize(file))
+            if upload_dir:
+                self.failUnless(response.context['file']['tmpfilename'])
+
+
     def test_redirect(self):
         "GET a URL that redirects elsewhere"
         response = self.client.get('/test_client/redirect_view/')

tests/modeltests/test_client/urls.py

@@ -4 +4 @@
 urlpatterns = patterns('',
     (r'^get_view/$', views.get_view),
     (r'^post_view/$', views.post_view),
+    (r'^post_file_view/$', views.post_file_view),
     (r'^raw_post_view/$', views.raw_post_view),
     (r'^redirect_view/$', views.redirect_view),
     (r'^form_view/$', views.form_view),

docs/request_response.txt

@@ -72 +72 @@
 ``FILES``
     A dictionary-like object containing all uploaded files. Each key in
     ``FILES`` is the ``name`` from the ``<input type="file" name="" />``. Each
-    value in ``FILES`` is a standard Python dictionary with the following three
+    value in ``FILES`` is a standard Python dictionary with the following four
     keys:
 
         * ``filename`` -- The name of the uploaded file, as a Python string.
         * ``content-type`` -- The content type of the uploaded file.
         * ``content`` -- The raw content of the uploaded file.
+        * ``content-length`` -- The length of the content in bytes.
 
+    If streaming file uploads are enabled two additional keys
+    describing the uploaded file will be present:
+
+        * ``tmpfilename`` -- The filename for the temporary file.
+        * ``tmpfile`` -- An open file object for the temporary file.
+
+    The temporary file will be removed when the request finishes.
+
+    Note that accessing ``content`` when streaming uploads are enabled
+    will read the whole file into memory which may not be what you want.
+
     Note that ``FILES`` will only contain data if the request method was POST
     and the ``<form>`` that posted to the request had
     ``enctype="multipart/form-data"``. Otherwise, ``FILES`` will be a blank

docs/settings.txt

@@ -437 +437 @@
 
 .. _Testing Django Applications: ../testing/
 
+FILE_UPLOAD_DIR
+---------------
+
+Default: ``None``
+
+Path to a directory where temporary files should be written during
+file uploads. Leaving this as ``None`` will disable streaming file uploads,
+and cause all uploaded files to be stored (temporarily) in memory.
+
 IGNORABLE_404_ENDS
 ------------------
 
@@ -780 +789 @@
 
 .. _site framework docs: ../sites/
 
+STREAMING_MIN_POST_SIZE
+-----------------------
+
+Default: 524288 (``512*1024``)
+
+An integer specifying the minimum number of bytes that has to be
+received (in a POST) for file upload streaming to take place. Any
+request smaller than this will be handled in memory. 
+Note: ``FILE_UPLOAD_DIR`` has to be defined to enable streaming.
+
 TEMPLATE_CONTEXT_PROCESSORS
 ---------------------------
 

docs/forms.txt

@@ -475 +475 @@
    new_data = request.POST.copy()
    new_data.update(request.FILES)
 
+Streaming file uploads.
+-----------------------
+
+File uploads will be read into memory by default. This works fine for
+small to medium sized uploads (from 1MB to 100MB depending on your
+setup and usage). If you want to support larger uploads you can enable
+upload streaming where only a small part of the file will be in memory
+at any time. To do this you need to specify the ``FILE_UPLOAD_DIR``
+setting (see the settings_ document for more details).
+
+See `request object`_ for more details about ``request.FILES`` objects
+with streaming file uploads enabled.
+
 Validators
 ==========
 
@@ -693 +706 @@
 .. _`generic views`: ../generic_views/
 .. _`models API`: ../model-api/
 .. _settings: ../settings/
+.. _request object: ../request_response/#httprequest-objects