Code

Ticket #2070: 2070_revision7599.diff

File 2070_revision7599.diff, 45.6 KB (added by leahculver, 6 years ago)

Updated patch to r7599

Line 
1Index: django/test/client.py
2===================================================================
3--- django/test/client.py       (revision 2453)
4+++ django/test/client.py       (working copy)
5@@ -19,6 +19,24 @@
6 BOUNDARY = 'BoUnDaRyStRiNg'
7 MULTIPART_CONTENT = 'multipart/form-data; boundary=%s' % BOUNDARY
8 
9+class FakePayload(object):
10+    """
11+    A wrapper around StringIO that restricts what can be read,
12+    since data from the network can't be seeked and cannot
13+    be read outside of its content length (or else we hang).
14+    """
15+    def __init__(self, content):
16+        self.__content = StringIO(content)
17+        self.__len = len(content)
18+
19+    def read(self, num_bytes=None):
20+        if num_bytes is None:
21+            num_bytes = self.__len or 1
22+        assert self.__len >= num_bytes, "Cannot read more than the available bytes from the HTTP incoming data."
23+        content = self.__content.read(num_bytes)
24+        self.__len -= num_bytes
25+        return content
26+
27 class ClientHandler(BaseHandler):
28     """
29     A HTTP Handler that can be used for testing purposes.
30@@ -236,7 +254,7 @@
31             'CONTENT_TYPE':   content_type,
32             'PATH_INFO':      urllib.unquote(path),
33             'REQUEST_METHOD': 'POST',
34-            'wsgi.input':     StringIO(post_data),
35+            'wsgi.input':     FakePayload(post_data),
36         }
37         r.update(extra)
38 
39@@ -280,4 +298,4 @@
40         """
41         session = __import__(settings.SESSION_ENGINE, {}, {}, ['']).SessionStore()
42         session.delete(session_key=self.cookies[settings.SESSION_COOKIE_NAME].value)
43-        self.cookies = SimpleCookie()
44+        self.cookies = SimpleCookie()
45\ No newline at end of file
46Index: django/http/__init__.py
47===================================================================
48--- django/http/__init__.py     (revision 2453)
49+++ django/http/__init__.py     (working copy)
50@@ -9,9 +9,9 @@
51 except ImportError:
52     from cgi import parse_qsl
53 
54-from django.utils.datastructures import MultiValueDict, FileDict
55+from django.utils.datastructures import MultiValueDict, ImmutableList
56 from django.utils.encoding import smart_str, iri_to_uri, force_unicode
57-
58+from django.http.multipartparser import MultiPartParser
59 from utils import *
60 
61 RESERVED_CHARS="!*'();:@&=+$,/?%#[]"
62@@ -25,6 +25,7 @@
63 
64     # The encoding used in GET/POST dicts. None means use default setting.
65     _encoding = None
66+    _upload_handlers = ()
67 
68     def __init__(self):
69         self.GET, self.POST, self.COOKIES, self.META, self.FILES = {}, {}, {}, {}, {}
70@@ -102,40 +103,40 @@
71 
72     encoding = property(_get_encoding, _set_encoding)
73 
74-def parse_file_upload(header_dict, post_data):
75-    """Returns a tuple of (POST QueryDict, FILES MultiValueDict)."""
76-    import email, email.Message
77-    from cgi import parse_header
78-    raw_message = '\r\n'.join(['%s:%s' % pair for pair in header_dict.items()])
79-    raw_message += '\r\n\r\n' + post_data
80-    msg = email.message_from_string(raw_message)
81-    POST = QueryDict('', mutable=True)
82-    FILES = MultiValueDict()
83-    for submessage in msg.get_payload():
84-        if submessage and isinstance(submessage, email.Message.Message):
85-            name_dict = parse_header(submessage['Content-Disposition'])[1]
86-            # name_dict is something like {'name': 'file', 'filename': 'test.txt'} for file uploads
87-            # or {'name': 'blah'} for POST fields
88-            # We assume all uploaded files have a 'filename' set.
89-            if 'filename' in name_dict:
90-                assert type([]) != type(submessage.get_payload()), "Nested MIME messages are not supported"
91-                if not name_dict['filename'].strip():
92-                    continue
93-                # IE submits the full path, so trim everything but the basename.
94-                # (We can't use os.path.basename because that uses the server's
95-                # directory separator, which may not be the same as the
96-                # client's one.)
97-                filename = name_dict['filename'][name_dict['filename'].rfind("\\")+1:]
98-                FILES.appendlist(name_dict['name'], FileDict({
99-                    'filename': filename,
100-                    'content-type': 'Content-Type' in submessage and submessage['Content-Type'] or None,
101-                    'content': submessage.get_payload(),
102-                }))
103-            else:
104-                POST.appendlist(name_dict['name'], submessage.get_payload())
105-    return POST, FILES
106+    def _initialize_handlers(self):
107+        from django.conf import settings
108+        from django.core.files.uploadhandler import load_handler
109+        handlers = []
110+        # We go through each handler in the settings variable
111+        # and instantiate the handler by calling HandlerClass(request).
112+        for handler in settings.FILE_UPLOAD_HANDLERS:
113+            handlers.append(load_handler(handler, self))
114+        self._upload_handlers = handlers
115 
116+    def _set_upload_handlers(self, upload_handlers):
117+        """
118+        Set the upload handler to the new handler given in the parameter.
119+        """
120+        if hasattr(self, '_files'):
121+            raise AttributeError("You cannot set the upload handlers after the upload has been processed.")
122+        self._upload_handlers = upload_handlers
123 
124+    def _get_upload_handlers(self):
125+        if not self._upload_handlers:
126+            # If thre are no upload handlers defined, initialize them from settings.
127+            self._initialize_handlers()
128+        return self._upload_handlers
129+
130+    upload_handlers = property(_get_upload_handlers, _set_upload_handlers)
131+
132+    def parse_file_upload(self, META, post_data):
133+        """Returns a tuple of (POST QueryDict, FILES MultiValueDict)."""
134+        self.upload_handlers = ImmutableList(self.upload_handlers,
135+                                             warning="You cannot alter the upload handlers after the upload has been processed.")
136+        parser = MultiPartParser(META, post_data, self.upload_handlers,
137+                                 self.encoding)
138+        return parser.parse()
139+
140 class QueryDict(MultiValueDict):
141     """
142     A specialized MultiValueDict that takes a query string when initialized.
143Index: django/oldforms/__init__.py
144===================================================================
145--- django/oldforms/__init__.py (revision 2453)
146+++ django/oldforms/__init__.py (working copy)
147@@ -680,18 +680,27 @@
148         self.field_name, self.is_required = field_name, is_required
149         self.validator_list = [self.isNonEmptyFile] + validator_list
150 
151-    def isNonEmptyFile(self, field_data, all_data):
152+    def isNonEmptyFile(self, new_data, all_data):
153+        if hasattr(new_data, 'upload_errors'):
154+            upload_errors = new_data.upload_errors()
155+            if upload_errors:
156+                raise validators.CriticalValidationError, upload_errors
157         try:
158-            content = field_data['content']
159-        except TypeError:
160-            raise validators.CriticalValidationError, ugettext("No file was submitted. Check the encoding type on the form.")
161-        if not content:
162+            file_size = new_data.file_size
163+        except AttributeError:
164+            file_size = len(new_data['content'])
165+        if not file_size:
166             raise validators.CriticalValidationError, ugettext("The submitted file is empty.")
167 
168     def render(self, data):
169         return mark_safe(u'<input type="file" id="%s" class="v%s" name="%s" />' % \
170             (self.get_id(), self.__class__.__name__, self.field_name))
171 
172+    def prepare(self, new_data):
173+        if hasattr(new_data, 'upload_errors'):
174+            upload_errors = new_data.upload_errors()
175+            new_data[self.field_name] = { '_file_upload_error': upload_errors }
176+
177     def html2python(data):
178         if data is None:
179             raise EmptyValue
180Index: django/db/models/base.py
181===================================================================
182--- django/db/models/base.py    (revision 2453)
183+++ django/db/models/base.py    (working copy)
184@@ -19,6 +19,7 @@
185 from django.utils.datastructures import SortedDict
186 from django.utils.functional import curry
187 from django.utils.encoding import smart_str, force_unicode, smart_unicode
188+from django.core.files.move import file_move_safe
189 from django.conf import settings
190 
191 try:
192@@ -447,12 +448,29 @@
193     def _get_FIELD_size(self, field):
194         return os.path.getsize(self._get_FIELD_filename(field))
195 
196-    def _save_FIELD_file(self, field, filename, raw_contents, save=True):
197+    def _save_FIELD_file(self, field, filename, raw_field, save=True):
198         directory = field.get_directory_name()
199         try: # Create the date-based directory if it doesn't exist.
200             os.makedirs(os.path.join(settings.MEDIA_ROOT, directory))
201         except OSError: # Directory probably already exists.
202             pass
203+
204+        # Put the deprecation warning first since there are multiple
205+        # locations where we use the new and old interface.
206+        if isinstance(raw_field, dict):
207+            import warnings
208+            from django.core.files.uploadedfile import SimpleUploadedFile
209+            raw_field = SimpleUploadedFile.from_dict(raw_field)
210+            warnings.warn("The dictionary usage for files is deprecated. Use the new object interface instead.", DeprecationWarning)
211+        elif isinstance(raw_field, basestring):
212+            import warnings
213+            from django.core.files.uploadedfile import SimpleUploadedFile
214+            raw_field = SimpleUploadedFile(filename, raw_field)
215+            warnings.warn("The string interface for save_FIELD_file is deprecated.", DeprecationWarning)
216+
217+        if filename is None:
218+            filename = raw_field.file_name
219+
220         filename = field.get_filename(filename)
221 
222         # If the filename already exists, keep adding an underscore to the name of
223@@ -469,9 +487,20 @@
224         setattr(self, field.attname, filename)
225 
226         full_filename = self._get_FIELD_filename(field)
227-        fp = open(full_filename, 'wb')
228-        fp.write(raw_contents)
229-        fp.close()
230+        if hasattr(raw_field, 'temporary_file_path'):
231+            # This file has a file path that we can move.
232+            raw_field.close()
233+            file_move_safe(raw_field.temporary_file_path(), full_filename)
234+        else:
235+            from django.core.files import locks
236+            fp = open(full_filename, 'wb')
237+            # exclusive lock
238+            locks.lock(fp, locks.LOCK_EX)
239+            # This is a normal uploadedfile that we can stream.
240+            for chunk in raw_field.chunk(65535):
241+                fp.write(chunk)
242+            locks.unlock(fp)
243+            fp.close()
244 
245         # Save the width and/or height, if applicable.
246         if isinstance(field, ImageField) and (field.width_field or field.height_field):
247Index: django/db/models/fields/__init__.py
248===================================================================
249--- django/db/models/fields/__init__.py (revision 2453)
250+++ django/db/models/fields/__init__.py (working copy)
251@@ -806,7 +806,7 @@
252         setattr(cls, 'get_%s_filename' % self.name, curry(cls._get_FIELD_filename, field=self))
253         setattr(cls, 'get_%s_url' % self.name, curry(cls._get_FIELD_url, field=self))
254         setattr(cls, 'get_%s_size' % self.name, curry(cls._get_FIELD_size, field=self))
255-        setattr(cls, 'save_%s_file' % self.name, lambda instance, filename, raw_contents, save=True: instance._save_FIELD_file(self, filename, raw_contents, save))
256+        setattr(cls, 'save_%s_file' % self.name, lambda instance, filename, raw_field, save=True: instance._save_FIELD_file(self, filename, raw_field, save))
257         dispatcher.connect(self.delete_file, signal=signals.post_delete, sender=cls)
258 
259     def delete_file(self, instance):
260@@ -829,10 +829,16 @@
261         if new_data.get(upload_field_name, False):
262             func = getattr(new_object, 'save_%s_file' % self.name)
263             if rel:
264-                func(new_data[upload_field_name][0]["filename"], new_data[upload_field_name][0]["content"], save)
265+                file = new_data[upload_field_name][0]
266             else:
267-                func(new_data[upload_field_name]["filename"], new_data[upload_field_name]["content"], save)
268+                file = new_data[upload_field_name]
269 
270+            try:
271+                file_name = file.file_name
272+            except AttributeError:
273+                file_name = file['filename']
274+            func(file_name, file, save)
275+
276     def get_directory_name(self):
277         return os.path.normpath(force_unicode(datetime.datetime.now().strftime(smart_str(self.upload_to))))
278 
279@@ -844,7 +850,7 @@
280     def save_form_data(self, instance, data):
281         from django.newforms.fields import UploadedFile
282         if data and isinstance(data, UploadedFile):
283-            getattr(instance, "save_%s_file" % self.name)(data.filename, data.content, save=False)
284+            getattr(instance, "save_%s_file" % self.name)(data.filename, data.data, save=False)
285 
286     def formfield(self, **kwargs):
287         defaults = {'form_class': forms.FileField}
288Index: django/conf/global_settings.py
289===================================================================
290--- django/conf/global_settings.py      (revision 2453)
291+++ django/conf/global_settings.py      (working copy)
292@@ -228,6 +228,22 @@
293 # Example: "http://media.lawrence.com"
294 MEDIA_URL = ''
295 
296+# A tuple that enumerates the upload handlers
297+# in order.
298+FILE_UPLOAD_HANDLERS = (
299+    'django.core.files.uploadhandler.MemoryFileUploadHandler',
300+    'django.core.files.uploadhandler.TemporaryFileUploadHandler',
301+)
302+
303+# Number of bytes the length of the request can be before it is
304+# streamed to the file system instead of parsed entirely in memory.
305+FILE_UPLOAD_MAX_MEMORY_SIZE = 2621440
306+
307+# Directory to upload streamed files temporarily.
308+# A value of `None` means that it will use the default temporary
309+# directory for the server's operating system.
310+FILE_UPLOAD_TEMP_DIR = None
311+
312 # Default formatting for date objects. See all available format strings here:
313 # http://www.djangoproject.com/documentation/templates/#now
314 DATE_FORMAT = 'N j, Y'
315Index: django/core/handlers/wsgi.py
316===================================================================
317--- django/core/handlers/wsgi.py        (revision 2453)
318+++ django/core/handlers/wsgi.py        (working copy)
319@@ -112,9 +112,8 @@
320         # Populates self._post and self._files
321         if self.method == 'POST':
322             if self.environ.get('CONTENT_TYPE', '').startswith('multipart'):
323-                header_dict = dict([(k, v) for k, v in self.environ.items() if k.startswith('HTTP_')])
324-                header_dict['Content-Type'] = self.environ.get('CONTENT_TYPE', '')
325-                self._post, self._files = http.parse_file_upload(header_dict, self.raw_post_data)
326+                self._raw_post_data = ''
327+                self._post, self._files = self.parse_file_upload(self.META, self.environ['wsgi.input'])
328             else:
329                 self._post, self._files = http.QueryDict(self.raw_post_data, encoding=self._encoding), datastructures.MultiValueDict()
330         else:
331Index: django/core/handlers/modpython.py
332===================================================================
333--- django/core/handlers/modpython.py   (revision 2453)
334+++ django/core/handlers/modpython.py   (working copy)
335@@ -53,7 +53,8 @@
336     def _load_post_and_files(self):
337         "Populates self._post and self._files"
338         if 'content-type' in self._req.headers_in and self._req.headers_in['content-type'].startswith('multipart'):
339-            self._post, self._files = http.parse_file_upload(self._req.headers_in, self.raw_post_data)
340+            self._raw_post_data = ''
341+            self._post, self._files = self.parse_file_upload(self.META, self._req)
342         else:
343             self._post, self._files = http.QueryDict(self.raw_post_data, encoding=self._encoding), datastructures.MultiValueDict()
344 
345Index: django/newforms/fields.py
346===================================================================
347--- django/newforms/fields.py   (revision 2453)
348+++ django/newforms/fields.py   (working copy)
349@@ -4,6 +4,7 @@
350 
351 import copy
352 import datetime
353+import warnings
354 import os
355 import re
356 import time
357@@ -416,9 +417,9 @@
358 
359 class UploadedFile(StrAndUnicode):
360     "A wrapper for files uploaded in a FileField"
361-    def __init__(self, filename, content):
362+    def __init__(self, filename, data):
363         self.filename = filename
364-        self.content = content
365+        self.data = data
366 
367     def __unicode__(self):
368         """
369@@ -444,16 +445,28 @@
370             return None
371         elif not data and initial:
372             return initial
373+
374+        if isinstance(data, dict):
375+            # We warn once, then support both ways below.
376+            warnings.warn("The dictionary usage for files is deprecated. Use the new object interface instead.", DeprecationWarning)
377+
378         try:
379-            f = UploadedFile(data['filename'], data['content'])
380-        except TypeError:
381+            file_name = data.file_name
382+            file_size = data.file_size
383+        except AttributeError:
384+            try:
385+                file_name = data.get('filename')
386+                file_size = bool(data['content'])
387+            except (AttributeError, KeyError):
388+                raise ValidationError(self.error_messages['invalid'])
389+
390+        if not file_name:
391             raise ValidationError(self.error_messages['invalid'])
392-        except KeyError:
393-            raise ValidationError(self.error_messages['missing'])
394-        if not f.content:
395+        if not file_size:
396             raise ValidationError(self.error_messages['empty'])
397-        return f
398 
399+        return UploadedFile(file_name, data)
400+
401 class ImageField(FileField):
402     default_error_messages = {
403         'invalid_image': _(u"Upload a valid image. The file you uploaded was either not an image or a corrupted image."),
404@@ -470,15 +483,36 @@
405         elif not data and initial:
406             return initial
407         from PIL import Image
408-        from cStringIO import StringIO
409+
410+        # We need to get the file, it either has a path
411+        # or we have to read it all into memory...
412+        if hasattr(data, 'temporary_file_path'):
413+            file = data.temporary_file_path()
414+        else:
415+            try:
416+                from cStringIO import StringIO
417+            except ImportError:
418+                from StringIO import StringIO
419+            if hasattr(data, 'read'):
420+                file = StringIO(data.read())
421+            else:
422+                file = StringIO(data['content'])
423+
424         try:
425             # load() is the only method that can spot a truncated JPEG,
426             #  but it cannot be called sanely after verify()
427-            trial_image = Image.open(StringIO(f.content))
428+            trial_image = Image.open(file)
429             trial_image.load()
430+
431+            # Since we're about to use the file again, we have to
432+            # reset the cursor of the file object if it has a cursor
433+            # to reset.
434+            if hasattr(file, 'reset'):
435+                file.reset()
436+
437             # verify() is the only method that can spot a corrupt PNG,
438             #  but it must be called immediately after the constructor
439-            trial_image = Image.open(StringIO(f.content))
440+            trial_image = Image.open(file)
441             trial_image.verify()
442         except Exception: # Python Imaging Library doesn't recognize it as an image
443             raise ValidationError(self.error_messages['invalid_image'])
444Index: django/utils/datastructures.py
445===================================================================
446--- django/utils/datastructures.py      (revision 2453)
447+++ django/utils/datastructures.py      (working copy)
448@@ -332,14 +332,34 @@
449             except TypeError: # Special-case if current isn't a dict.
450                 current = {bits[-1]: v}
451 
452-class FileDict(dict):
453+class ImmutableList(tuple):
454     """
455-    A dictionary used to hold uploaded file contents. The only special feature
456-    here is that repr() of this object won't dump the entire contents of the
457-    file to the output. A handy safeguard for a large file upload.
458+    A tuple-like object that raises useful
459+    errors when it is asked to mutate.
460+    Example::
461+        a = ImmutableList(range(5), warning=AttributeError("You cannot mutate this."))
462+        a[3] = '4'
463+        (Raises the AttributeError)
464     """
465-    def __repr__(self):
466-        if 'content' in self:
467-            d = dict(self, content='<omitted>')
468-            return dict.__repr__(d)
469-        return dict.__repr__(self)
470+
471+    def __new__(cls, *args, **kwargs):
472+        if 'warning' in kwargs:
473+            warning = kwargs['warning']
474+            del kwargs['warning']
475+        else:
476+            warning = 'ImmutableList object is immutable.'
477+        self = tuple.__new__(cls, *args, **kwargs)
478+        self.warning = warning
479+        return self
480+
481+    def complain(self, *wargs, **kwargs):
482+        if isinstance(self.warning, Exception):
483+            raise self.warning
484+        else:
485+            raise AttributeError, self.warning
486+
487+    # All list mutation functions become complain.
488+    __delitem__ = __delslice__ = __iadd__ = __imul__ = complain
489+    __setitem__ = __setslice__ = complain
490+    append = extend = insert = pop = remove = complain
491+    sort = reverse = complain
492Index: django/utils/text.py
493===================================================================
494--- django/utils/text.py        (revision 2453)
495+++ django/utils/text.py        (working copy)
496@@ -3,6 +3,7 @@
497 from django.utils.encoding import force_unicode
498 from django.utils.functional import allow_lazy
499 from django.utils.translation import ugettext_lazy
500+from htmlentitydefs import name2codepoint
501 
502 # Capitalizes the first letter of a string.
503 capfirst = lambda x: x and force_unicode(x)[0].upper() + force_unicode(x)[1:]
504@@ -222,3 +223,26 @@
505             yield bit
506 smart_split = allow_lazy(smart_split, unicode)
507 
508+def _replace_entity(match):
509+     text = match.group(1)
510+     if text[0] == u'#':
511+         text = text[1:]
512+         try:
513+             if text[0] in u'xX':
514+                 c = int(text[1:], 16)
515+             else:
516+                 c = int(text)
517+             return unichr(c)
518+         except ValueError:
519+             return match.group(0)
520+     else:
521+         try:
522+             return unichr(name2codepoint[text])
523+         except (ValueError, KeyError):
524+             return match.group(0)
525+
526+_entity_re = re.compile(r"&(#?[xX]?(?:[0-9a-fA-F]+|\w{1,8}));")
527+
528+def unescape_entities(text):
529+     return _entity_re.sub(_replace_entity, text)
530+unescape_entities = allow_lazy(unescape_entities, unicode)
531Index: tests/modeltests/model_forms/models.py
532===================================================================
533--- tests/modeltests/model_forms/models.py      (revision 2453)
534+++ tests/modeltests/model_forms/models.py      (working copy)
535@@ -67,7 +67,13 @@
536 
537 class ImageFile(models.Model):
538     description = models.CharField(max_length=20)
539-    image = models.FileField(upload_to=tempfile.gettempdir())
540+    try:
541+        # If PIL is available, try testing PIL.
542+        # Otherwise, it's equivalent to TextFile above.
543+        import Image
544+        image = models.ImageField(upload_to=tempfile.gettempdir())
545+    except ImportError:
546+        image = models.FileField(upload_to=tempfile.gettempdir())
547 
548     def __unicode__(self):
549         return self.description
550@@ -75,6 +81,9 @@
551 __test__ = {'API_TESTS': """
552 >>> from django import newforms as forms
553 >>> from django.newforms.models import ModelForm
554+>>> from django.core.files.uploadedfile import SimpleUploadedFile
555+>>> from warnings import filterwarnings
556+>>> filterwarnings("ignore")
557 
558 The bare bones, absolutely nothing custom, basic case.
559 
560@@ -792,6 +801,17 @@
561 
562 # Upload a file and ensure it all works as expected.
563 
564+>>> f = TextFileForm(data={'description': u'Assistance'}, files={'file': SimpleUploadedFile('test1.txt', 'hello world')})
565+>>> f.is_valid()
566+True
567+>>> type(f.cleaned_data['file'])
568+<class 'django.newforms.fields.UploadedFile'>
569+>>> instance = f.save()
570+>>> instance.file
571+u'...test1.txt'
572+
573+>>> os.unlink(instance.get_file_filename())
574+
575 >>> f = TextFileForm(data={'description': u'Assistance'}, files={'file': {'filename': 'test1.txt', 'content': 'hello world'}})
576 >>> f.is_valid()
577 True
578@@ -814,18 +834,30 @@
579 u'...test1.txt'
580 
581 # Delete the current file since this is not done by Django.
582-
583 >>> os.unlink(instance.get_file_filename())
584 
585 # Override the file by uploading a new one.
586 
587->>> f = TextFileForm(data={'description': u'Assistance'}, files={'file': {'filename': 'test2.txt', 'content': 'hello world'}}, instance=instance)
588+>>> f = TextFileForm(data={'description': u'Assistance'}, files={'file': SimpleUploadedFile('test2.txt', 'hello world')}, instance=instance)
589 >>> f.is_valid()
590 True
591 >>> instance = f.save()
592 >>> instance.file
593 u'...test2.txt'
594 
595+# Delete the current file since this is not done by Django.
596+>>> os.unlink(instance.get_file_filename())
597+
598+>>> f = TextFileForm(data={'description': u'Assistance'}, files={'file': {'filename': 'test2.txt', 'content': 'hello world'}})
599+>>> f.is_valid()
600+True
601+>>> instance = f.save()
602+>>> instance.file
603+u'...test2.txt'
604+
605+# Delete the current file since this is not done by Django.
606+>>> os.unlink(instance.get_file_filename())
607+
608 >>> instance.delete()
609 
610 # Test the non-required FileField
611@@ -838,14 +870,28 @@
612 >>> instance.file
613 ''
614 
615->>> f = TextFileForm(data={'description': u'Assistance'}, files={'file': {'filename': 'test3.txt', 'content': 'hello world'}}, instance=instance)
616+>>> f = TextFileForm(data={'description': u'Assistance'}, files={'file': SimpleUploadedFile('test3.txt', 'hello world')}, instance=instance)
617 >>> f.is_valid()
618 True
619 >>> instance = f.save()
620 >>> instance.file
621 u'...test3.txt'
622+
623+# Delete the current file since this is not done by Django.
624+>>> os.unlink(instance.get_file_filename())
625 >>> instance.delete()
626 
627+>>> f = TextFileForm(data={'description': u'Assistance'}, files={'file': {'filename': 'test3.txt', 'content': 'hello world'}})
628+>>> f.is_valid()
629+True
630+>>> instance = f.save()
631+>>> instance.file
632+u'...test3.txt'
633+
634+# Delete the current file since this is not done by Django.
635+>>> os.unlink(instance.get_file_filename())
636+>>> instance.delete()
637+
638 # ImageField ###################################################################
639 
640 # ImageField and FileField are nearly identical, but they differ slighty when
641@@ -858,6 +904,18 @@
642 
643 >>> image_data = open(os.path.join(os.path.dirname(__file__), "test.png")).read()
644 
645+>>> f = ImageFileForm(data={'description': u'An image'}, files={'image': SimpleUploadedFile('test.png', image_data)})
646+>>> f.is_valid()
647+True
648+>>> type(f.cleaned_data['image'])
649+<class 'django.newforms.fields.UploadedFile'>
650+>>> instance = f.save()
651+>>> instance.image
652+u'...test.png'
653+
654+# Delete the current file since this is not done by Django.
655+>>> os.unlink(instance.get_image_filename())
656+
657 >>> f = ImageFileForm(data={'description': u'An image'}, files={'image': {'filename': 'test.png', 'content': image_data}})
658 >>> f.is_valid()
659 True
660@@ -885,15 +943,28 @@
661 
662 # Override the file by uploading a new one.
663 
664->>> f = ImageFileForm(data={'description': u'Changed it'}, files={'image': {'filename': 'test2.png', 'content': image_data}}, instance=instance)
665+>>> f = ImageFileForm(data={'description': u'Changed it'}, files={'image': SimpleUploadedFile('test2.png', image_data)}, instance=instance)
666 >>> f.is_valid()
667 True
668 >>> instance = f.save()
669 >>> instance.image
670 u'...test2.png'
671 
672+# Delete the current file since this is not done by Django.
673+>>> os.unlink(instance.get_image_filename())
674 >>> instance.delete()
675 
676+>>> f = ImageFileForm(data={'description': u'Changed it'}, files={'image': {'filename': 'test2.png', 'content': image_data}})
677+>>> f.is_valid()
678+True
679+>>> instance = f.save()
680+>>> instance.image
681+u'...test2.png'
682+
683+# Delete the current file since this is not done by Django.
684+>>> os.unlink(instance.get_image_filename())
685+>>> instance.delete()
686+
687 # Test the non-required ImageField
688 
689 >>> f = ImageFileForm(data={'description': u'Test'})
690@@ -904,12 +975,23 @@
691 >>> instance.image
692 ''
693 
694->>> f = ImageFileForm(data={'description': u'And a final one'}, files={'image': {'filename': 'test3.png', 'content': image_data}}, instance=instance)
695+>>> f = ImageFileForm(data={'description': u'And a final one'}, files={'image': SimpleUploadedFile('test3.png', image_data)}, instance=instance)
696 >>> f.is_valid()
697 True
698 >>> instance = f.save()
699 >>> instance.image
700 u'...test3.png'
701+
702+# Delete the current file since this is not done by Django.
703+>>> os.unlink(instance.get_image_filename())
704 >>> instance.delete()
705 
706+>>> f = ImageFileForm(data={'description': u'And a final one'}, files={'image': {'filename': 'test3.png', 'content': image_data}})
707+>>> f.is_valid()
708+True
709+>>> instance = f.save()
710+>>> instance.image
711+u'...test3.png'
712+>>> instance.delete()
713+
714 """}
715Index: tests/regressiontests/bug639/tests.py
716===================================================================
717--- tests/regressiontests/bug639/tests.py       (revision 2453)
718+++ tests/regressiontests/bug639/tests.py       (working copy)
719@@ -9,6 +9,7 @@
720 from regressiontests.bug639.models import Photo
721 from django.http import QueryDict
722 from django.utils.datastructures import MultiValueDict
723+from django.core.files.uploadedfile import SimpleUploadedFile
724 
725 class Bug639Test(unittest.TestCase):
726         
727@@ -21,12 +22,8 @@
728         
729         # Fake a request query dict with the file
730         qd = QueryDict("title=Testing&image=", mutable=True)
731-        qd["image_file"] = {
732-            "filename" : "test.jpg",
733-            "content-type" : "image/jpeg",
734-            "content" : img
735-        }
736-       
737+        qd["image_file"] = SimpleUploadedFile('test.jpg', img, 'image/jpeg')
738+
739         manip = Photo.AddManipulator()
740         manip.do_html2python(qd)
741         p = manip.save(qd)
742@@ -39,4 +36,4 @@
743         Make sure to delete the "uploaded" file to avoid clogging /tmp.
744         """
745         p = Photo.objects.get()
746-        os.unlink(p.get_image_filename())
747\ No newline at end of file
748+        os.unlink(p.get_image_filename())
749Index: tests/regressiontests/forms/error_messages.py
750===================================================================
751--- tests/regressiontests/forms/error_messages.py       (revision 2453)
752+++ tests/regressiontests/forms/error_messages.py       (working copy)
753@@ -1,6 +1,7 @@
754 # -*- coding: utf-8 -*-
755 tests = r"""
756 >>> from django.newforms import *
757+>>> from django.core.files.uploadedfile import SimpleUploadedFile
758 
759 # CharField ###################################################################
760 
761@@ -214,11 +215,11 @@
762 Traceback (most recent call last):
763 ...
764 ValidationError: [u'INVALID']
765->>> f.clean({})
766+>>> f.clean(SimpleUploadedFile('name', None))
767 Traceback (most recent call last):
768 ...
769-ValidationError: [u'MISSING']
770->>> f.clean({'filename': 'name', 'content':''})
771+ValidationError: [u'EMPTY FILE']
772+>>> f.clean(SimpleUploadedFile('name', ''))
773 Traceback (most recent call last):
774 ...
775 ValidationError: [u'EMPTY FILE']
776Index: tests/regressiontests/forms/tests.py
777===================================================================
778--- tests/regressiontests/forms/tests.py        (revision 2453)
779+++ tests/regressiontests/forms/tests.py        (working copy)
780@@ -26,6 +26,8 @@
781 from regressions import tests as regression_tests
782 from util import tests as util_tests
783 from widgets import tests as widgets_tests
784+from warnings import filterwarnings
785+filterwarnings("ignore")
786 
787 __test__ = {
788     'extra_tests': extra_tests,
789Index: tests/regressiontests/forms/fields.py
790===================================================================
791--- tests/regressiontests/forms/fields.py       (revision 2453)
792+++ tests/regressiontests/forms/fields.py       (working copy)
793@@ -2,6 +2,7 @@
794 tests = r"""
795 >>> from django.newforms import *
796 >>> from django.newforms.widgets import RadioFieldRenderer
797+>>> from django.core.files.uploadedfile import SimpleUploadedFile
798 >>> import datetime
799 >>> import time
800 >>> import re
801@@ -773,12 +774,12 @@
802 >>> f.clean({})
803 Traceback (most recent call last):
804 ...
805-ValidationError: [u'No file was submitted.']
806+ValidationError: [u'No file was submitted. Check the encoding type on the form.']
807 
808 >>> f.clean({}, '')
809 Traceback (most recent call last):
810 ...
811-ValidationError: [u'No file was submitted.']
812+ValidationError: [u'No file was submitted. Check the encoding type on the form.']
813 
814 >>> f.clean({}, 'files/test3.pdf')
815 'files/test3.pdf'
816@@ -788,20 +789,20 @@
817 ...
818 ValidationError: [u'No file was submitted. Check the encoding type on the form.']
819 
820->>> f.clean({'filename': 'name', 'content': None})
821+>>> f.clean(SimpleUploadedFile('name', None))
822 Traceback (most recent call last):
823 ...
824 ValidationError: [u'The submitted file is empty.']
825 
826->>> f.clean({'filename': 'name', 'content': ''})
827+>>> f.clean(SimpleUploadedFile('name', ''))
828 Traceback (most recent call last):
829 ...
830 ValidationError: [u'The submitted file is empty.']
831 
832->>> type(f.clean({'filename': 'name', 'content': 'Some File Content'}))
833+>>> type(f.clean(SimpleUploadedFile('name', 'Some File Content')))
834 <class 'django.newforms.fields.UploadedFile'>
835 
836->>> type(f.clean({'filename': 'name', 'content': 'Some File Content'}, 'files/test4.pdf'))
837+>>> type(f.clean(SimpleUploadedFile('name', 'Some File Content'), 'files/test4.pdf'))
838 <class 'django.newforms.fields.UploadedFile'>
839 
840 # URLField ##################################################################
841Index: tests/regressiontests/forms/forms.py
842===================================================================
843--- tests/regressiontests/forms/forms.py        (revision 2453)
844+++ tests/regressiontests/forms/forms.py        (working copy)
845@@ -1,6 +1,7 @@
846 # -*- coding: utf-8 -*-
847 tests = r"""
848 >>> from django.newforms import *
849+>>> from django.core.files.uploadedfile import SimpleUploadedFile
850 >>> import datetime
851 >>> import time
852 >>> import re
853@@ -1465,7 +1466,7 @@
854 >>> print f
855 <tr><th>File1:</th><td><ul class="errorlist"><li>This field is required.</li></ul><input type="file" name="file1" /></td></tr>
856 
857->>> f = FileForm(data={}, files={'file1': {'filename': 'name', 'content':''}}, auto_id=False)
858+>>> f = FileForm(data={}, files={'file1': SimpleUploadedFile('name', '')}, auto_id=False)
859 >>> print f
860 <tr><th>File1:</th><td><ul class="errorlist"><li>The submitted file is empty.</li></ul><input type="file" name="file1" /></td></tr>
861 
862@@ -1473,7 +1474,7 @@
863 >>> print f
864 <tr><th>File1:</th><td><ul class="errorlist"><li>No file was submitted. Check the encoding type on the form.</li></ul><input type="file" name="file1" /></td></tr>
865 
866->>> f = FileForm(data={}, files={'file1': {'filename': 'name', 'content':'some content'}}, auto_id=False)
867+>>> f = FileForm(data={}, files={'file1': SimpleUploadedFile('name', 'some content')}, auto_id=False)
868 >>> print f
869 <tr><th>File1:</th><td><input type="file" name="file1" /></td></tr>
870 >>> f.is_valid()
871Index: tests/regressiontests/test_client_regress/views.py
872===================================================================
873--- tests/regressiontests/test_client_regress/views.py  (revision 2453)
874+++ tests/regressiontests/test_client_regress/views.py  (working copy)
875@@ -1,4 +1,5 @@
876 import os
877+import sha
878 
879 from django.contrib.auth.decorators import login_required
880 from django.http import HttpResponse, HttpResponseRedirect, HttpResponseServerError
881@@ -13,9 +14,10 @@
882     Check that a file upload can be updated into the POST dictionary without
883     going pear-shaped.
884     """
885+    from django.core.files.uploadedfile import UploadedFile
886     form_data = request.POST.copy()
887     form_data.update(request.FILES)
888-    if isinstance(form_data['file_field'], dict) and isinstance(form_data['name'], unicode):
889+    if isinstance(form_data.get('file_field'), UploadedFile) and isinstance(form_data['name'], unicode):
890         # If a file is posted, the dummy client should only post the file name,
891         # not the full path.
892         if os.path.dirname(form_data['file_field']['filename']) != '':
893@@ -24,13 +26,40 @@
894     else:
895         return HttpResponseServerError()
896 
897+def file_upload_view_verify(request):
898+    """
899+    Use the sha digest hash to verify the uploaded contents.
900+    """
901+    from django.core.files.uploadedfile import UploadedFile
902+    form_data = request.POST.copy()
903+    form_data.update(request.FILES)
904+
905+    # Check to see if unicode names worked out.
906+    if not request.FILES['file_unicode'].file_name.endswith(u'test_\u4e2d\u6587_Orl\xe9ans.jpg'):
907+        return HttpResponseServerError()
908+
909+    for key, value in form_data.items():
910+        if key.endswith('_hash'):
911+            continue
912+        if key + '_hash' not in form_data:
913+            continue
914+        submitted_hash = form_data[key + '_hash']
915+        if isinstance(value, UploadedFile):
916+            new_hash = sha.new(value.read()).hexdigest()
917+        else:
918+            new_hash = sha.new(value).hexdigest()
919+        if new_hash != submitted_hash:
920+            return HttpResponseServerError()
921+
922+    return HttpResponse('')
923+
924 def staff_only_view(request):
925     "A view that can only be visited by staff. Non staff members get an exception"
926     if request.user.is_staff:
927         return HttpResponse('')
928     else:
929         raise SuspiciousOperation()
930-   
931+
932 def get_view(request):
933     "A simple login protected view"
934     return HttpResponse("Hello world")
935Index: tests/regressiontests/test_client_regress/models.py
936===================================================================
937--- tests/regressiontests/test_client_regress/models.py (revision 2453)
938+++ tests/regressiontests/test_client_regress/models.py (working copy)
939@@ -6,6 +6,7 @@
940 from django.core.urlresolvers import reverse
941 from django.core.exceptions import SuspiciousOperation
942 import os
943+import sha
944 
945 class AssertContainsTests(TestCase):
946     def test_contains(self):
947@@ -250,6 +251,50 @@
948         response = self.client.post('/test_client_regress/file_upload/', post_data)
949         self.assertEqual(response.status_code, 200)
950 
951+    def test_large_upload(self):
952+        import tempfile
953+        dir = tempfile.gettempdir()
954+
955+        (fd, name1) = tempfile.mkstemp(suffix='.file1', dir=dir)
956+        file1 = os.fdopen(fd, 'w+b')
957+        file1.write('a' * (2 ** 21))
958+        file1.seek(0)
959+
960+        (fd, name2) = tempfile.mkstemp(suffix='.file2', dir=dir)
961+        file2 = os.fdopen(fd, 'w+b')
962+        file2.write('a' * (10 * 2 ** 20))
963+        file2.seek(0)
964+
965+        # This file contains chinese symbols for a name.
966+        name3 = os.path.join(dir, u'test_&#20013;&#25991;_Orl\u00e9ans.jpg')
967+        file3 = open(name3, 'w+b')
968+        file3.write('b' * (2 ** 10))
969+        file3.seek(0)
970+
971+        post_data = {
972+            'name': 'Ringo',
973+            'file_field1': file1,
974+            'file_field2': file2,
975+            'file_unicode': file3,
976+            }
977+
978+        for key in post_data.keys():
979+            try:
980+                post_data[key + '_hash'] = sha.new(post_data[key].read()).hexdigest()
981+                post_data[key].seek(0)
982+            except AttributeError:
983+                post_data[key + '_hash'] = sha.new(post_data[key]).hexdigest()
984+
985+        response = self.client.post('/test_client_regress/file_upload_verify/', post_data)
986+
987+        for name in (name1, name2, name3):
988+            try:
989+                os.unlink(name)
990+            except:
991+                pass
992+
993+        self.assertEqual(response.status_code, 200)
994+
995 class LoginTests(TestCase):
996     fixtures = ['testdata']
997 
998Index: tests/regressiontests/test_client_regress/urls.py
999===================================================================
1000--- tests/regressiontests/test_client_regress/urls.py   (revision 2453)
1001+++ tests/regressiontests/test_client_regress/urls.py   (working copy)
1002@@ -4,6 +4,7 @@
1003 urlpatterns = patterns('',
1004     (r'^no_template_view/$', views.no_template_view),
1005     (r'^file_upload/$', views.file_upload_view),
1006+    (r'^file_upload_verify/$', views.file_upload_view_verify),
1007     (r'^staff_only/$', views.staff_only_view),
1008     (r'^get_view/$', views.get_view),
1009     url(r'^arg_view/(?P<name>.+)/$', views.view_with_argument, name='arg_view'),
1010Index: tests/regressiontests/datastructures/tests.py
1011===================================================================
1012--- tests/regressiontests/datastructures/tests.py       (revision 2453)
1013+++ tests/regressiontests/datastructures/tests.py       (working copy)
1014@@ -117,12 +117,23 @@
1015 >>> d['person']['2']['firstname']
1016 ['Adrian']
1017 
1018-### FileDict ################################################################
1019-
1020->>> d = FileDict({'content': 'once upon a time...'})
1021+### ImmutableList ################################################################
1022+>>> d = ImmutableList(range(10))
1023+>>> d.sort()
1024+Traceback (most recent call last):
1025+  File "<stdin>", line 1, in <module>
1026+  File "/var/lib/python-support/python2.5/django/utils/datastructures.py", line 359, in complain
1027+    raise AttributeError, self.warning
1028+AttributeError: ImmutableList object is immutable.
1029 >>> repr(d)
1030-"{'content': '<omitted>'}"
1031->>> d = FileDict({'other-key': 'once upon a time...'})
1032->>> repr(d)
1033-"{'other-key': 'once upon a time...'}"
1034+'(0, 1, 2, 3, 4, 5, 6, 7, 8, 9)'
1035+>>> d = ImmutableList(range(10), warning="Object is immutable!")
1036+>>> d[1]
1037+1
1038+>>> d[1] = 'test'
1039+Traceback (most recent call last):
1040+  File "<stdin>", line 1, in <module>
1041+  File "/var/lib/python-support/python2.5/django/utils/datastructures.py", line 359, in complain
1042+    raise AttributeError, self.warning
1043+AttributeError: Object is immutable!
1044 """
1045Index: AUTHORS
1046===================================================================
1047--- AUTHORS     (revision 2453)
1048+++ AUTHORS     (working copy)
1049@@ -59,7 +59,7 @@
1050     Arthur <avandorp@gmail.com>
1051     av0000@mail.ru
1052     David Avsajanishvili <avsd05@gmail.com>
1053-    axiak@mit.edu
1054+    Mike Axiak <axiak@mit.edu>
1055     Niran Babalola <niran@niran.org>
1056     Morten Bagai <m@bagai.com>
1057     Mikaël Barbero <mikael.barbero nospam at nospam free.fr>
1058@@ -139,6 +139,7 @@
1059     Marc Fargas <telenieko@telenieko.com>
1060     Szilveszter Farkas <szilveszter.farkas@gmail.com>
1061     favo@exoweb.net
1062+    fdr <drfarina@gmail.com>
1063     Dmitri Fedortchenko <zeraien@gmail.com>
1064     Liang Feng <hutuworm@gmail.com>
1065     Bill Fenner <fenner@gmail.com>
1066Index: docs/request_response.txt
1067===================================================================
1068--- docs/request_response.txt   (revision 2453)
1069+++ docs/request_response.txt   (working copy)
1070@@ -80,20 +80,24 @@
1071     strings.
1072 
1073 ``FILES``
1074+   **New in Django development version**
1075     A dictionary-like object containing all uploaded files. Each key in
1076     ``FILES`` is the ``name`` from the ``<input type="file" name="" />``. Each
1077-    value in ``FILES`` is a standard Python dictionary with the following three
1078-    keys:
1079+    value in ``FILES`` is an ``UploadedFile`` object containing at least the
1080+    following attributes:
1081 
1082-        * ``filename`` -- The name of the uploaded file, as a Python string.
1083-        * ``content-type`` -- The content type of the uploaded file.
1084-        * ``content`` -- The raw content of the uploaded file.
1085+        * ``read(num_bytes=None)`` -- Read a number of bytes from the file.
1086+        * ``file_name`` -- The name of the uploaded file.
1087+        * ``file_size`` -- The size, in bytes, of the uploaded file.
1088+       * ``chunk()`` -- A generator that yields sequential chunks of data.
1089 
1090-    Note that ``FILES`` will only contain data if the request method was POST
1091-    and the ``<form>`` that posted to the request had
1092-    ``enctype="multipart/form-data"``. Otherwise, ``FILES`` will be a blank
1093-    dictionary-like object.
1094+    See `File Uploads`_ for more information. Note that ``FILES`` will only
1095+    contain data if the request method was POST and the ``<form>`` that posted
1096+    to the request had ``enctype="multipart/form-data"``. Otherwise, ``FILES``
1097+    will be a blank dictionary-like object.
1098 
1099+    .. _File Uploads: ../upload_handling/
1100+
1101 ``META``
1102     A standard Python dictionary containing all available HTTP headers.
1103     Available headers depend on the client and server, but here are some
1104Index: docs/settings.txt
1105===================================================================
1106--- docs/settings.txt   (revision 2453)
1107+++ docs/settings.txt   (working copy)
1108@@ -279,7 +279,7 @@
1109 
1110 The database backend to use. The build-in database backends are
1111 ``'postgresql_psycopg2'``, ``'postgresql'``, ``'mysql'``, ``'mysql_old'``,
1112-``'sqlite3'`` and ``'oracle'``.
1113+``'sqlite3'``, ``'oracle'``, and ``'oracle'``.
1114 
1115 In the Django development version, you can use a database backend that doesn't
1116 ship with Django by setting ``DATABASE_ENGINE`` to a fully-qualified path (i.e.
1117@@ -523,6 +523,36 @@
1118 The character encoding used to decode any files read from disk. This includes
1119 template files and initial SQL data files.
1120 
1121+FILE_UPLOAD_HANDLERS
1122+--------------------
1123+
1124+**New in Django development version**
1125+
1126+Default::
1127+
1128+    ("django.core.files.fileuploadhandler.MemoryFileUploadHandler",
1129+     "django.core.files.fileuploadhandler.TemporaryFileUploadHandler",)
1130+
1131+A tuple of handlers to use for uploading.
1132+
1133+FILE_UPLOAD_MAX_MEMORY_SIZE
1134+---------------------------
1135+
1136+**New in Django development version**
1137+
1138+Default: ``2621440``
1139+
1140+The maximum size (in bytes) that an upload will be before it gets streamed to the file system.
1141+
1142+FILE_UPLOAD_TEMP_DIR
1143+--------------------
1144+
1145+**New in Django development version**
1146+
1147+Default: ``None``
1148+
1149+The directory to store data temporarily while uploading files. If ``None``, Django will use the standard temporary directory for the operating system. For example, this will default to '/tmp' on *nix-style operating systems.
1150+
1151 FIXTURE_DIRS
1152 -------------
1153 
1154Index: docs/newforms.txt
1155===================================================================
1156--- docs/newforms.txt   (revision 2453)
1157+++ docs/newforms.txt   (working copy)
1158@@ -805,12 +805,12 @@
1159 need to bind the file data containing the mugshot image::
1160 
1161     # Bound form with an image field
1162+    >>> from django.core.files.uploadedfile import SimpleUploadedFile
1163     >>> data = {'subject': 'hello',
1164     ...         'message': 'Hi there',
1165     ...         'sender': 'foo@example.com',
1166     ...         'cc_myself': True}
1167-    >>> file_data = {'mugshot': {'filename':'face.jpg'
1168-    ...                          'content': <file data>}}
1169+    >>> file_data = {'mugshot': SimpleUploadedFile('face.jpg', <file data>)}
1170     >>> f = ContactFormWithMugshot(data, file_data)
1171 
1172 In practice, you will usually specify ``request.FILES`` as the source