Ticket #2070: 5722.diff

django/http/__init__.py

@@ -4 +4 @@
-from urllib import urlencode
-from django.utils.datastructures import MultiValueDict
-from django.utils.encoding import smart_str, iri_to_uri, force_unicode
+from django.http.multipartparser import MultiPartParser, MultiPartParserError 
+import re
-
+upload_id_re = re.compile(r'^[a-fA-F0-9]{32}$') # file progress id Regular expression 
+
-RESERVED_CHARS="!*'();:@&=+$,/?%#[]"
-
-try:
@@ -64 +68 @@
-
-    encoding = property(_get_encoding, _set_encoding)
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
+
+
-
+    def _set_file_progress(self,value):
+        pass
+
+    def _del_file_progress(self):
+        pass
+
+    file_progress = property(_get_file_progress,
+                             _set_file_progress,
+                             _del_file_progress)
+
+    def _get_file_progress_from_args(self, headers, get, querystring): 
+        """
+        This parses the request for a file progress_id value. 
+        Note that there are two distinct ways of getting the progress 
+        ID -- header and GET. One is used primarily to attach via JavaScript 
+        to the end of an HTML form action while the other is used for AJAX 
+        communication. 
+        
+        All progress IDs must be valid 32-digit hexadecimal numbers. 
+        """ 
+        if 'X-Upload-ID' in headers: 
+            progress_id = headers['X-Upload-ID'] 
+        elif 'progress_id' in get: 
+            progress_id = get['progress_id'] 
+        else: 
+            return None 
+        
+        if not upload_id_re.match(progress_id): 
+            return None 
+
+        return progress_id 
+
+def parse_file_upload(headers, input, request): 
+    from django.conf import settings 
+
+    # Only stream files to disk if FILE_STREAMING_DIR is set 
+    file_upload_dir = settings.FILE_UPLOAD_DIR 
+    streaming_min_post_size = settings.STREAMING_MIN_POST_SIZE 
+    
+    try: 
+        parser = MultiPartParser(headers, input, request, file_upload_dir, streaming_min_post_size) 
+        return parser.parse() 
+    except MultiPartParserError, e:
+        return MultiValueDict({ '_file_upload_error': [e.message] }), {}
+
+
-class QueryDict(MultiValueDict):
-    """
-    A specialized MultiValueDict that takes a query string when initialized.

django/oldforms/__init__.py

@@ -676 +676 @@
-        self.validator_list = [self.isNonEmptyFile] + validator_list
-
-    def isNonEmptyFile(self, field_data, all_data):
-try
-            content = field_data['content
-        except TypeError
+if field_data.has_key('_file_upload_error')
+           raise validators.CriticalValidationError, field_data['_file_upload_error
+       if not field_data.has_key('filename')
-            raise validators.CriticalValidationError, ugettext("No file was submitted. Check the encoding type on the form.")
-content
+field_data['content-length']
-            raise validators.CriticalValidationError, ugettext("The submitted file is empty.")
-
-    def render(self, data):
-        return u'<input type="file" id="%s" class="v%s" name="%s" />' % \
-            (self.get_id(), self.__class__.__name__, self.field_name)
+    
+    def prepare(self, new_data): 
+        if new_data.has_key('_file_upload_error'): 
+            # pretend we got something in the field to raise a validation error later 
+            new_data[self.field_name] = { '_file_upload_error': new_data['_file_upload_error'] } 
-
-    def html2python(data):
-        if data is None:

django/db/models/base.py

@@ -13 +13 @@
-from django.utils.datastructures import SortedDict
-from django.utils.functional import curry
-from django.utils.encoding import smart_str, force_unicode
+from django.utils.file import file_move_safe
-from django.conf import settings
-from itertools import izip
-import types
@@ -365 +366 @@
-    def _get_FIELD_size(self, field):
-        return os.path.getsize(self._get_FIELD_filename(field))
-
-contents
+field
-        directory = field.get_directory_name()
-        try: # Create the date-based directory if it doesn't exist.
-            os.makedirs(os.path.join(settings.MEDIA_ROOT, directory))
-        except OSError: # Directory probably already exists.
-            pass
+
+        if filename is None:
+            filename = raw_field['filename']
+
-        filename = field.get_filename(filename)
-
-        # If the filename already exists, keep adding an underscore to the name of
@@ -387 +392 @@
-        setattr(self, field.attname, filename)
-
-        full_filename = self._get_FIELD_filename(field)
-
-
-
+
+
+
+
+
+
+
+
+
+
-
-        # Save the width and/or height, if applicable.
-        if isinstance(field, ImageField) and (field.width_field or field.height_field):

django/db/models/fields/__init__.py

@@ -707 +707 @@
-        setattr(cls, 'get_%s_filename' % self.name, curry(cls._get_FIELD_filename, field=self))
-        setattr(cls, 'get_%s_url' % self.name, curry(cls._get_FIELD_url, field=self))
-        setattr(cls, 'get_%s_size' % self.name, curry(cls._get_FIELD_size, field=self))
-
+
+
-        dispatcher.connect(self.delete_file, signal=signals.post_delete, sender=cls)
-
-    def delete_file(self, instance):
@@ -730 +731 @@
-        if new_data.get(upload_field_name, False):
-            func = getattr(new_object, 'save_%s_file' % self.name)
-            if rel:
-["content"]
+
-            else:
-["content"]
+
-
-    def get_directory_name(self):
-        return os.path.normpath(force_unicode(datetime.datetime.now().strftime(smart_str(self.upload_to))))

django/conf/global_settings.py

@@ -247 +247 @@
-from django import get_version
-URL_VALIDATOR_USER_AGENT = "Django/%s (http://www.djangoproject.com)" % get_version()
-
+# The directory to place streamed file uploads. The web server needs write
+# permissions on this directory.
+# If this is None, streaming uploads are disabled.
+FILE_UPLOAD_DIR = None
+
+# The minimum size of a POST before file uploads are streamed to disk.
+# Any less than this number, and the file is uploaded to memory.
+# Size is in bytes.
+STREAMING_MIN_POST_SIZE = 512 * (2**10)
+
-##############
-# MIDDLEWARE #
-##############

django/core/handlers/wsgi.py

@@ -76 +76 @@
-        self.environ = environ
-        self.path = force_unicode(environ['PATH_INFO'])
-        self.META = environ
+        self.META['UPLOAD_PROGRESS_ID'] = self._get_file_progress_id()
-        self.method = environ['REQUEST_METHOD'].upper()
-
-    def __repr__(self):
@@ -112 +113 @@
-            if self.environ.get('CONTENT_TYPE', '').startswith('multipart'):
-                header_dict = dict([(k, v) for k, v in self.environ.items() if k.startswith('HTTP_')])
-                header_dict['Content-Type'] = self.environ.get('CONTENT_TYPE', '')
-
+
+
+
+
+
+
+
+
-            else:
-                self._post, self._files = http.QueryDict(self.raw_post_data, encoding=self._encoding), datastructures.MultiValueDict()
-        else:
@@ -168 +176 @@
-            buf.close()
-            return self._raw_post_data
-
+    def _get_file_progress_id(self):
+        """
+        Returns the Progress ID of the request,
+        usually provided if there is a file upload
+        going on.
+        Returns ``None`` if no progress ID is specified.
+        """
+        return self._get_file_progress_from_args(self.environ,
+                                                 self.GET,
+                                                 self.environ.get('QUERY_STRING', ''))
+
-    GET = property(_get_get, _set_get)
-    POST = property(_get_post, _set_post)
-    COOKIES = property(_get_cookies, _set_cookies)

django/core/handlers/modpython.py

@@ -48 +48 @@
-    def _load_post_and_files(self):
-        "Populates self._post and self._files"
-        if 'content-type' in self._req.headers_in and self._req.headers_in['content-type'].startswith('multipart'):
-
+
+
+
+
+
+
-        else:
-            self._post, self._files = http.QueryDict(self.raw_post_data, encoding=self._encoding), datastructures.MultiValueDict()
-
@@ -93 +98 @@
-                'AUTH_TYPE':         self._req.ap_auth_type,
-                'CONTENT_LENGTH':    self._req.clength, # This may be wrong
-                'CONTENT_TYPE':      self._req.content_type, # This may be wrong
-
-
-
-
-
-
-
-
-
-
-
-
-
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
-            }
-            for key, value in self._req.headers_in.items():
-                key = 'HTTP_' + key.upper().replace('-', '_')
@@ -123 +129 @@
-    def _get_method(self):
-        return self.META['REQUEST_METHOD'].upper()
-
+    def _get_file_progress_id(self):
+        """
+        Returns the Progress ID of the request,
+        usually provided if there is a file upload
+        going on.
+        Returns ``None`` if no progress ID is specified.
+        """
+        return self._get_file_progress_from_args(self._req.headers_in,
+                                                 self.GET,
+                                                 self._req.args)
+
-    GET = property(_get_get, _set_get)
-    POST = property(_get_post, _set_post)
-    COOKIES = property(_get_cookies, _set_cookies)

django/newforms/fields.py

@@ -110 +110 @@
-        super(CharField, self).clean(value)
-        if value in EMPTY_VALUES:
-            return u''
+        if isinstance(value, dict):
+            return value
-        value = smart_unicode(value)
-        value_length = len(value)
-        if self.max_length is not None and value_length > self.max_length:

django/contrib/admin/urls.py

@@ -10 +10 @@
-    ('^$', 'django.contrib.admin.views.main.index'),
-    ('^r/(\d+)/(.*)/$', 'django.views.defaults.shortcut'),
-    ('^jsi18n/$', i18n_view, {'packages': 'django.conf'}),
+    ('^upload_progress/$', 'django.contrib.admin.views.main.upload_progress'),
-    ('^logout/$', 'django.contrib.auth.views.logout'),
-    ('^password_change/$', 'django.contrib.auth.views.password_change'),
-    ('^password_change/done/$', 'django.contrib.auth.views.password_change_done'),

django/contrib/admin/views/main.py

@@ -9 +9 @@
-from django.shortcuts import get_object_or_404, render_to_response
-from django.db import models
-from django.db.models.query import handle_legacy_orderlist, QuerySet
-
+, HttpResponseServerError
-from django.utils.html import escape
-from django.utils.text import capfirst, get_text_list
-from django.utils.encoding import force_unicode, smart_str
@@ -88 +88 @@
-def get_javascript_imports(opts, auto_populated_fields, field_sets):
-# Put in any necessary JavaScript imports.
-    js = ['js/core.js', 'js/admin/RelatedObjectLookups.js']
+    if opts.has_field_type(models.FileField) and settings.FILE_UPLOAD_DIR:
+        js.append('js/UploadProgress.js')
-    if auto_populated_fields:
-        js.append('js/urlify.js')
-    if opts.has_field_type(models.DateTimeField) or opts.has_field_type(models.TimeField) or opts.has_field_type(models.DateField):
@@ -789 +791 @@
-                               'admin/%s/change_list.html' % app_label,
-                               'admin/change_list.html'], context_instance=c)
-change_list = staff_member_required(never_cache(change_list))
+
+def upload_progress(request):
+    """
+    Given this request, returns a JSON
+    object that has information on a file upload progress.
+    If there is no file upload in progress, returns an
+    empty dictionary, '{}'.
+    """
+    from django.utils import simplejson
+
+    content = simplejson.dumps(request.file_progress)
+
+    if content.strip() == '{}':
+        return HttpResponseServerError('')
+    else:
+        return HttpResponse(content=content, mimetype='text/plain')

django/contrib/admin/templates/admin/change_form.html

@@ -65 +65 @@
-   {% auto_populated_field_script auto_populated_fields change %}
-   </script>
-{% endif %}
+
+{% if has_file_field %}
+<div id="progress_wrap" style="position: absolute; background: white; z-index: 9040; display: none; visibility: hidden; width: 420px; height: 50px padding: 10px; border: solid 1px #ddd;">
+   <a href="#" onclick="close_progress();return false" title="Close Progress Bar" 
+      style="color: #c00; font-size: 1.5em; font-weight: bold; float: right; padding: 0; position: relative; top: -2px; left: -2px;">X</a>
+   <h1>Upload progress</h1>
+
+   <div id="progress_bar" style="top: 0; left: 0; width: 0; z-index: 9049; height: 4px;" class="submit-row"></div>
+   <div id="progress_text" style="color: black;">0%</div>
-</div>
+{% endif %}
+
+</div>
-</form></div>
-{% endblock %}

tests/modeltests/test_client/views.py

@@ -46 +46 @@
-
-    return HttpResponse(t.render(c))
-
+def post_file_view(request):
+    "A view that expects a multipart post and returns a file in the context"
+    t = Template('File {{ file.filename }} received', name='POST Template')
+    c = Context({'file': request.FILES['file_file']})
+    return HttpResponse(t.render(c))
+
-def redirect_view(request):
-    "A view that redirects all requests to the GET view"
-    return HttpResponseRedirect('/test_client/get_view/')

tests/modeltests/test_client/models.py

@@ -4 +4 @@
-
-The test client is a class that can act like a simple
-browser for testing purposes.
-
+
-It allows the user to compose GET and POST requests, and
-obtain the response that the server gave to those requests.
-The server Response objects are annotated with the details
@@ -80 +80 @@
-        self.assertEqual(response.template.name, "Book template")
-        self.assertEqual(response.content, "Blink - Malcolm Gladwell")
-
+    def test_post_file_view(self):
+        "POST this python file to a view"
+        import os, tempfile
+        from django.conf import settings
+        file = __file__.replace('.pyc', '.py')
+        for upload_dir in [None, tempfile.gettempdir()]:
+            settings.FILE_UPLOAD_DIR = upload_dir
+            post_data = { 'name': file, 'file': open(file) }
+            response = self.client.post('/test_client/post_file_view/', post_data)
+            self.failUnless('models.py' in response.context['file']['filename'])
+            self.failUnless(len(response.context['file']['content']) == os.path.getsize(file))
+            if upload_dir:
+                self.failUnless(response.context['file']['tmpfilename'])
+
-    def test_redirect(self):
-        "GET a URL that redirects elsewhere"
-        response = self.client.get('/test_client/redirect_view/')

tests/modeltests/test_client/urls.py

@@ -5 +5 @@
-urlpatterns = patterns('',
-    (r'^get_view/$', views.get_view),
-    (r'^post_view/$', views.post_view),
+    (r'^post_file_view/$', views.post_file_view),
-    (r'^raw_post_view/$', views.raw_post_view),
-    (r'^redirect_view/$', views.redirect_view),
-    (r'^permanent_redirect_view/$', redirect_to, { 'url': '/test_client/get_view/' }),

docs/request_response.txt

@@ -72 +72 @@
-``FILES``
-    A dictionary-like object containing all uploaded files. Each key in
-    ``FILES`` is the ``name`` from the ``<input type="file" name="" />``. Each
-three
+four
-    keys:
-
-        * ``filename`` -- The name of the uploaded file, as a Python string.
-        * ``content-type`` -- The content type of the uploaded file.
-        * ``content`` -- The raw content of the uploaded file.
+        * ``content-length`` -- The length of the content in bytes.
-
+    If streaming file uploads are enabled two additional keys
+    describing the uploaded file will be present:
+
+        * ``tmpfilename`` -- The filename for the temporary file.
+        * ``tmpfile`` -- An open file object for the temporary file.
+
+    The temporary file will be removed when the request finishes.
+
+    Note that accessing ``content`` when streaming uploads are enabled
+    will read the whole file into memory which may not be what you want.
+
-    Note that ``FILES`` will only contain data if the request method was POST
-    and the ``<form>`` that posted to the request had
-    ``enctype="multipart/form-data"``. Otherwise, ``FILES`` will be a blank

docs/settings.txt

@@ -472 +472 @@
-
-.. _Testing Django Applications: ../testing/
-
+FILE_UPLOAD_DIR
+---------------
+
+Default: ``None``
+
+Path to a directory where temporary files should be written during
+file uploads. Leaving this as ``None`` will disable streaming file uploads,
+and cause all uploaded files to be stored (temporarily) in memory.
+
-IGNORABLE_404_ENDS
-------------------
-
@@ -788 +797 @@
-
-.. _site framework docs: ../sites/
-
+STREAMING_MIN_POST_SIZE
+-----------------------
+
+Default: 524288 (``512*1024``)
+
+An integer specifying the minimum number of bytes that has to be
+received (in a POST) for file upload streaming to take place. Any
+request smaller than this will be handled in memory. 
+Note: ``FILE_UPLOAD_DIR`` has to be defined to enable streaming.
+
-TEMPLATE_CONTEXT_PROCESSORS
----------------------------
-

docs/forms.txt

@@ -475 +475 @@
-   new_data = request.POST.copy()
-   new_data.update(request.FILES)
-
+Streaming file uploads.
+-----------------------
+
+File uploads will be read into memory by default. This works fine for
+small to medium sized uploads (from 1MB to 100MB depending on your
+setup and usage). If you want to support larger uploads you can enable
+upload streaming where only a small part of the file will be in memory
+at any time. To do this you need to specify the ``FILE_UPLOAD_DIR``
+setting (see the settings_ document for more details).
+
+See `request object`_ for more details about ``request.FILES`` objects
+with streaming file uploads enabled.
+
-Validators
-==========
-
@@ -698 +711 @@
-.. _`generic views`: ../generic_views/
-.. _`models API`: ../model-api/
-.. _settings: ../settings/
+.. _request object: ../request_response/#httprequest-objects