Index: django/http/__init__.py
===================================================================
--- django/http/__init__.py (revision 5099)
+++ django/http/__init__.py (working copy)
@@ -1,9 +1,14 @@
-import os
+import os, pickle
from Cookie import SimpleCookie
from pprint import pformat
from urllib import urlencode, quote
from django.utils.datastructures import MultiValueDict
+try:
+ from cStringIO import StringIO
+except ImportError:
+ from StringIO import StringIO
+
RESERVED_CHARS="!*'();:@&=+$,/?%#[]"
try:
@@ -306,3 +311,4 @@
if not host:
host = request.META.get('HTTP_HOST', '')
return host
+
Index: django/oldforms/__init__.py
===================================================================
--- django/oldforms/__init__.py (revision 5099)
+++ django/oldforms/__init__.py (working copy)
@@ -666,17 +666,22 @@
self.validator_list = [self.isNonEmptyFile] + validator_list
def isNonEmptyFile(self, field_data, all_data):
- try:
- content = field_data['content']
- except TypeError:
+ if field_data.has_key('_file_upload_error'):
+ raise validators.CriticalValidationError, field_data['_file_upload_error']
+ if not field_data.has_key('filename'):
raise validators.CriticalValidationError, gettext("No file was submitted. Check the encoding type on the form.")
- if not content:
+ if not field_data['content-length']:
raise validators.CriticalValidationError, gettext("The submitted file is empty.")
def render(self, data):
return '' % \
(self.get_id(), self.__class__.__name__, self.field_name)
+ def prepare(self, new_data):
+ if new_data.has_key('_file_upload_error'):
+ # pretend we got something in the field to raise a validation error later
+ new_data[self.field_name] = { '_file_upload_error': new_data['_file_upload_error'] }
+
def html2python(data):
if data is None:
raise EmptyValue
Index: django/db/models/base.py
===================================================================
--- django/db/models/base.py (revision 5099)
+++ django/db/models/base.py (working copy)
@@ -12,12 +12,14 @@
from django.dispatch import dispatcher
from django.utils.datastructures import SortedDict
from django.utils.functional import curry
+from django.utils.file import file_move_safe
from django.conf import settings
from itertools import izip
import types
import sys
import os
+
class ModelBase(type):
"Metaclass for all models"
def __new__(cls, name, bases, attrs):
@@ -361,7 +363,7 @@
def _get_FIELD_size(self, field):
return os.path.getsize(self._get_FIELD_filename(field))
- def _save_FIELD_file(self, field, filename, raw_contents, save=True):
+ def _save_FIELD_file(self, field, filename, raw_field, save=True):
directory = field.get_directory_name()
try: # Create the date-based directory if it doesn't exist.
os.makedirs(os.path.join(settings.MEDIA_ROOT, directory))
@@ -383,9 +385,13 @@
setattr(self, field.attname, filename)
full_filename = self._get_FIELD_filename(field)
- fp = open(full_filename, 'wb')
- fp.write(raw_contents)
- fp.close()
+ if raw_field.has_key('tmpfilename'):
+ raw_field['tmpfile'].close()
+ file_move_safe(raw_field['tmpfilename'], full_filename)
+ else:
+ fp = open(full_filename, 'wb')
+ fp.write(raw_field['content'])
+ fp.close()
# Save the width and/or height, if applicable.
if isinstance(field, ImageField) and (field.width_field or field.height_field):
Index: django/db/models/fields/__init__.py
===================================================================
--- django/db/models/fields/__init__.py (revision 5099)
+++ django/db/models/fields/__init__.py (working copy)
@@ -636,7 +636,7 @@
setattr(cls, 'get_%s_filename' % self.name, curry(cls._get_FIELD_filename, field=self))
setattr(cls, 'get_%s_url' % self.name, curry(cls._get_FIELD_url, field=self))
setattr(cls, 'get_%s_size' % self.name, curry(cls._get_FIELD_size, field=self))
- setattr(cls, 'save_%s_file' % self.name, lambda instance, filename, raw_contents, save=True: instance._save_FIELD_file(self, filename, raw_contents, save))
+ setattr(cls, 'save_%s_file' % self.name, lambda instance, filename, raw_field, save=True: instance._save_FIELD_file(self, filename, raw_field, save))
dispatcher.connect(self.delete_file, signal=signals.post_delete, sender=cls)
def delete_file(self, instance):
@@ -659,9 +659,9 @@
if new_data.get(upload_field_name, False):
func = getattr(new_object, 'save_%s_file' % self.name)
if rel:
- func(new_data[upload_field_name][0]["filename"], new_data[upload_field_name][0]["content"], save)
+ func(new_data[upload_field_name][0]["filename"], new_data[upload_field_name][0], save)
else:
- func(new_data[upload_field_name]["filename"], new_data[upload_field_name]["content"], save)
+ func(new_data[upload_field_name]["filename"], new_data[upload_field_name], save)
def get_directory_name(self):
return os.path.normpath(datetime.datetime.now().strftime(self.upload_to))
Index: django/conf/global_settings.py
===================================================================
--- django/conf/global_settings.py (revision 5099)
+++ django/conf/global_settings.py (working copy)
@@ -240,6 +240,20 @@
# isExistingURL validator.
URL_VALIDATOR_USER_AGENT = "Django/0.96pre (http://www.djangoproject.com)"
+# The directory to place streamed file uploads. The web server needs write
+# permissions on this directory.
+# If this is None, streaming uploads are disabled.
+FILE_UPLOAD_DIR = None
+
+
+# The minimum size of a POST before file uploads are streamed to disk.
+# Any less than this number, and the file is uploaded to memory.
+# Size is in bytes.
+STREAMING_MIN_POST_SIZE = 512 * (2**10)
+
+
+
+
##############
# MIDDLEWARE #
##############
@@ -335,3 +349,5 @@
# The list of directories to search for fixtures
FIXTURE_DIRS = ()
+
+
Index: django/core/handlers/wsgi.py
===================================================================
--- django/core/handlers/wsgi.py (revision 5099)
+++ django/core/handlers/wsgi.py (working copy)
@@ -111,7 +111,13 @@
if self.environ.get('CONTENT_TYPE', '').startswith('multipart'):
header_dict = dict([(k, v) for k, v in self.environ.items() if k.startswith('HTTP_')])
header_dict['Content-Type'] = self.environ.get('CONTENT_TYPE', '')
- self._post, self._files = http.parse_file_upload(header_dict, self.raw_post_data)
+ header_dict['Content-Length'] = self.environ.get('CONTENT_LENGTH', '')
+ try:
+ self._post, self._files = http.parse_file_upload(header_dict, self.environ['wsgi.input'], self.META.get('QUERY_STRING'))
+ except:
+ self._post, self._files = {}, {} # make sure we dont read the input stream again
+ raise
+ self._raw_post_data = None # raw data is not available for streamed multipart messages
else:
self._post, self._files = http.QueryDict(self.raw_post_data), datastructures.MultiValueDict()
else:
Index: django/core/handlers/modpython.py
===================================================================
--- django/core/handlers/modpython.py (revision 5099)
+++ django/core/handlers/modpython.py (working copy)
@@ -47,7 +47,12 @@
def _load_post_and_files(self):
"Populates self._post and self._files"
if 'content-type' in self._req.headers_in and self._req.headers_in['content-type'].startswith('multipart'):
- self._post, self._files = http.parse_file_upload(self._req.headers_in, self.raw_post_data)
+ self._raw_post_data = None # raw data is not available for streamed multipart messages
+ try:
+ self._post, self._files = http.parse_file_upload(self._req.headers_in, self._req, self.META.get('QUERY_STRING'))
+ except:
+ self._post, self._files = {}, {} # make sure we dont read the input stream again
+ raise
else:
self._post, self._files = http.QueryDict(self.raw_post_data), datastructures.MultiValueDict()
Index: django/utils/file.py
===================================================================
--- django/utils/file.py (revision 0)
+++ django/utils/file.py (revision 0)
@@ -0,0 +1,36 @@
+import os
+
+try:
+ import shutils
+ file_move = shutils.move
+except:
+ file_move = os.rename
+
+def file_move_safe(old_file_name, new_file_name, chunk_size = 1024*64):
+ """
+ Moves a file from one location to another in the safest way possible.
+
+ First, it tries using shutils.move, which is OS-dependent but doesn't
+ break with change of filesystems. Then it tries os.rename, which will
+ break if it encounters a change in filesystems. Lastly, it streams
+ it manually from one file to another in python.
+ """
+
+ try:
+ file_move(old_file_name, new_file_name)
+ return
+ except:
+ pass
+
+ new_file = open(new_file_name, 'wb')
+ old_file = open(old_file_name, 'rb')
+ current_chunk = None
+
+ while current_chunk != '':
+ current_chunk = old_file.read(chunk_size)
+ new_file.write(current_chunk)
+
+ new_file.close()
+ old_file.close()
+
+ os.remove(old_file_name)
Index: tests/modeltests/test_client/views.py
===================================================================
--- tests/modeltests/test_client/views.py (revision 5099)
+++ tests/modeltests/test_client/views.py (working copy)
@@ -44,6 +44,12 @@
return HttpResponse(t.render(c))
+def post_file_view(request):
+ "A view that expects a multipart post and returns a file in the context"
+ t = Template('File {{ file.filename }} received', name='POST Template')
+ c = Context({'file': request.FILES['file_file']})
+ return HttpResponse(t.render(c))
+
def redirect_view(request):
"A view that redirects all requests to the GET view"
return HttpResponseRedirect('/test_client/get_view/')
Index: tests/modeltests/test_client/models.py
===================================================================
--- tests/modeltests/test_client/models.py (revision 5099)
+++ tests/modeltests/test_client/models.py (working copy)
@@ -75,6 +75,21 @@
self.assertEqual(response.template.name, "Book template")
self.assertEqual(response.content, "Blink - Malcolm Gladwell")
+ def test_post_file_view(self):
+ "POST this python file to a view"
+ import os, tempfile
+ from django.conf import settings
+ file = __file__.replace('.pyc', '.py')
+ for upload_dir in [None, tempfile.gettempdir()]:
+ settings.FILE_UPLOAD_DIR = upload_dir
+ post_data = { 'name': file, 'file': open(file) }
+ response = self.client.post('/test_client/post_file_view/', post_data)
+ self.failUnless('models.py' in response.context['file']['filename'])
+ self.failUnless(len(response.context['file']['content']) == os.path.getsize(file))
+ if upload_dir:
+ self.failUnless(response.context['file']['tmpfilename'])
+
+
def test_redirect(self):
"GET a URL that redirects elsewhere"
response = self.client.get('/test_client/redirect_view/')
Index: tests/modeltests/test_client/urls.py
===================================================================
--- tests/modeltests/test_client/urls.py (revision 5099)
+++ tests/modeltests/test_client/urls.py (working copy)
@@ -4,6 +4,7 @@
urlpatterns = patterns('',
(r'^get_view/$', views.get_view),
(r'^post_view/$', views.post_view),
+ (r'^post_file_view/$', views.post_file_view),
(r'^raw_post_view/$', views.raw_post_view),
(r'^redirect_view/$', views.redirect_view),
(r'^form_view/$', views.form_view),
Index: docs/request_response.txt
===================================================================
--- docs/request_response.txt (revision 5099)
+++ docs/request_response.txt (working copy)
@@ -72,13 +72,25 @@
``FILES``
A dictionary-like object containing all uploaded files. Each key in
``FILES`` is the ``name`` from the ````. Each
- value in ``FILES`` is a standard Python dictionary with the following three
+ value in ``FILES`` is a standard Python dictionary with the following four
keys:
* ``filename`` -- The name of the uploaded file, as a Python string.
* ``content-type`` -- The content type of the uploaded file.
* ``content`` -- The raw content of the uploaded file.
+ * ``content-length`` -- The length of the content in bytes.
+ If streaming file uploads are enabled two additional keys
+ describing the uploaded file will be present:
+
+ * ``tmpfilename`` -- The filename for the temporary file.
+ * ``tmpfile`` -- An open file object for the temporary file.
+
+ The temporary file will be removed when the request finishes.
+
+ Note that accessing ``content`` when streaming uploads are enabled
+ will read the whole file into memory which may not be what you want.
+
Note that ``FILES`` will only contain data if the request method was POST
and the ``