Code

Ticket #2070: 2070_revision7363.diff

File 2070_revision7363.diff, 93.2 KB (added by axiak, 6 years ago)

Altered documentation to be more approachable.

Line 
1Index: django/http/multipartparser.py
2===================================================================
3--- django/http/multipartparser.py      (revision 0)
4+++ django/http/multipartparser.py      (revision 0)
5@@ -0,0 +1,640 @@
6+"""
7+MultiPart parsing for file uploads.
8+
9+This object will take the file upload headers
10+and the file upload handler and chunk the upload
11+data for the handler to deal with.
12+"""
13+from django.utils.datastructures import MultiValueDict
14+from django.utils.encoding import force_unicode
15+from django.utils.text import unescape_entities
16+
17+__all__ = ('MultiPartParser','MultiPartParserError','InputStreamExhausted')
18+
19+class MultiPartParserError(Exception):
20+    pass
21+
22+class InputStreamExhausted(Exception):
23+    """ No more reads are allowed from this device. """
24+    pass
25+
26+class MultiPartParser(object):
27+    """
28+    A rfc2388 multipart/form-data parser.
29+
30+    parse() reads the input stream in chunk_size chunks and returns a
31+    tuple of (POST MultiValueDict, FILES MultiValueDict). If
32+    file_upload_dir is defined files will be streamed to temporary
33+    files in the specified directory.
34+    """
35+    def __init__(self, META, input_data, upload_handlers, encoding=None):
36+        """
37+        Initialize the MultiPartParser object.
38+
39+        *META* -- The standard META dictionary in Django request objects.
40+        *input_data* -- The raw post data, as a bytestring.
41+        *upload_handler* -- An object of type UploadHandler
42+                            that performs operations on the uploaded
43+                            data.
44+        *encoding* -- The encoding with which to treat the incoming data.
45+        """
46+        # Import cgi utilities for (near) future use.
47+        global valid_boundary, settings
48+        from django.conf import settings
49+        from cgi import valid_boundary
50+
51+        #
52+        # Content-Type should containt multipart and the boundary information.
53+        #
54+
55+        content_type = META.get('HTTP_CONTENT_TYPE', META.get('CONTENT_TYPE', ''))
56+        if not content_type.startswith('multipart/'):
57+            raise MultiPartParserError('Invalid Content-Type: %s' %
58+                                       content_type)
59+
60+        # Parse the header to get the boundary to split the parts.
61+        ctypes, opts = parse_header(content_type)
62+        boundary = opts.get('boundary')
63+        if not boundary or not valid_boundary(boundary):
64+            raise MultiPartParserError('Invalid boundary in multipart: %s' %
65+                                       boundary)
66+
67+
68+        #
69+        # Content-Length should contain the length of the body we are about
70+        # to receive.
71+        #
72+        try:
73+            content_length = int(META.get('HTTP_CONTENT_LENGTH',
74+                                          META.get('CONTENT_LENGTH',0)))
75+        except (ValueError, TypeError):
76+            # For now set it to 0...we'll try again later on down.
77+            content_length = 0
78+
79+        if content_length <= 0:
80+            # This means we shouldn't continue...raise an error.
81+            raise MultiPartParserError("Invalid content length: %r" % content_length)
82+
83+        self._boundary = boundary
84+        self._input_data = input_data
85+
86+        # For compatibility with low-level network APIs (with 32-bit integers),
87+        # the chunk size should be < 2^31, but still divisible by 4.
88+        self._chunk_size = min(2147483644, *[x.chunk_size for x in upload_handlers
89+                                            if x.chunk_size])
90+
91+        self._meta = META
92+        self._encoding = encoding or settings.DEFAULT_CHARSET
93+        self._content_length = content_length
94+        self._upload_handlers = upload_handlers
95+
96+    def parse(self):
97+        """
98+        Parse the POST data and break it into a FILES MultiValueDict
99+        and a POST MultiValueDict.
100+
101+           *returns* -- A tuple containing the POST and FILES dictionary,
102+                        respectively.
103+        """
104+        from django.core.files.fileuploadhandler import StopUpload, SkipFile
105+        from django.http import QueryDict
106+
107+        encoding = self._encoding
108+        handlers = self._upload_handlers
109+
110+        limited_input_data = LimitBytes(self._input_data, self._content_length)
111+
112+        # See if the handler will want to take care of the parsing.
113+        # This allows overriding everything if somebody wants it.
114+        for handler in handlers:
115+            result = handler.handle_raw_input(limited_input_data,
116+                                              self._meta,
117+                                              self._content_length,
118+                                              self._boundary,
119+                                              encoding)
120+            if result is not None:
121+                return result[0], result[1]
122+
123+        # Create the data structures to be used later.
124+        self._post = QueryDict('', mutable=True)
125+        self._files = MultiValueDict()
126+
127+        # Instantiate the parser and stream:
128+        stream = LazyStream(ChunkIter(limited_input_data, self._chunk_size))
129+
130+        # Whether or not to signal a file-completion at the beginning of the loop.
131+        old_field_name = None
132+        counters = [0] * len(handlers)
133+
134+        for item_type, meta_data, stream in Parser(stream, self._boundary):
135+            if old_field_name:
136+                # We run this at the beginning of the next loop
137+                # since we cannot be sure a file is complete until
138+                # we hit the next boundary/part of the multipart content.
139+                self.handle_file_complete(old_field_name, counters)
140+
141+            try:
142+                disposition = meta_data['content-disposition'][1]
143+                field_name = disposition['name'].strip()
144+            except (KeyError, IndexError, AttributeError):
145+                continue
146+
147+            transfer_encoding = meta_data.get('content-transfer-encoding')
148+            field_name = force_unicode(field_name, encoding, errors='replace')
149+
150+            if item_type == 'FIELD':
151+                # This is a post field, we can just set it in the post
152+                if transfer_encoding == 'base64':
153+                    raw_data = stream.read()
154+                    try:
155+                        data = str(raw_data).decode('base64')
156+                    except:
157+                        data = raw_data
158+                else:
159+                    data = stream.read()
160+
161+                self._post.appendlist(field_name,
162+                                      force_unicode(data, encoding, errors='replace'))
163+            elif item_type == 'FILE':
164+                # This is a file, use the handler...
165+                file_successful = True
166+                file_name = disposition.get('filename')
167+                if not file_name:
168+                    continue
169+                file_name = force_unicode(file_name, encoding, errors='replace')
170+                file_name = self.IE_sanitize(unescape_entities(file_name))
171+
172+
173+                content_type = meta_data.get('content-type', ('',))[0].strip()
174+                try:
175+                    charset = meta_data.get('content-type', (0,{}))[1].get('charset', None)
176+                except:
177+                    charset = None
178+
179+                try:
180+                    content_length = int(meta_data.get('content-length')[0])
181+                except (IndexError, TypeError, ValueError):
182+                    content_length = None
183+
184+                counters = [0] * len(handlers)
185+                try:
186+                    for handler in handlers:
187+                        retval = handler.new_file(field_name, file_name,
188+                                                  content_type, content_length,
189+                                                  charset)
190+                        if retval:
191+                            break
192+
193+                    for chunk in stream:
194+                        if transfer_encoding == 'base64':
195+                            # We only special-case base64 transfer encoding
196+                            try:
197+                                chunk = str(chunk).decode('base64')
198+                            except Exception, e:
199+                                # Since this is only a chunk, any error is an unfixable error.
200+                                raise MultiValueParseError("Could not decode base64 data: %r" % e)
201+
202+                        for i, handler in enumerate(handlers):
203+                            chunk_length = len(chunk)
204+                            counters[i] += chunk_length
205+                            chunk = handler.receive_data_chunk(chunk,
206+                                                               counters[i] - chunk_length,
207+                                                               counters[i])
208+                            if chunk is None:
209+                                # If the chunk received by the handler is None, then don't continue.
210+                                break
211+
212+                except (StopUpload, SkipFile), e:
213+                    file_successful = False
214+                    if isinstance(e, SkipFile):
215+                        # Just use up the rest of this file...
216+                        stream.exhaust()
217+                    elif isinstance(e, StopUpload):
218+                        # Abort the parsing and break
219+                        parser.abort()
220+                        break
221+                else:
222+                    # Handle file upload completions on next iteration.
223+                    old_field_name = field_name
224+            else:
225+                # If this is neither a FIELD or a FILE, just exhaust the stream.
226+                stream.exhuast()
227+
228+        # Make sure that the request data is all fed
229+        limited_input_data.exhaust()
230+        # Signal that the upload has completed.
231+        for handler in handlers:
232+            retval = handler.upload_complete()
233+            if retval:
234+                break
235+
236+        return self._post, self._files
237+
238+    def handle_file_complete(self, old_field_name, counters):
239+        """
240+        Handle all the signalling that takes place when a file is complete.
241+        """
242+        for i, handler in enumerate(self._upload_handlers):
243+            file_obj = handler.file_complete(counters[i])
244+            if file_obj:
245+                # If it returns a file object, then set the files dict.
246+                self._files.appendlist(force_unicode(old_field_name,
247+                                                     self._encoding,
248+                                                     errors='replace'),
249+                                       file_obj)
250+                break
251+
252+    def IE_sanitize(self, filename):
253+        """cleanup filename from IE full paths"""
254+        return filename and filename[filename.rfind("\\")+1:].strip()
255+
256+
257+class LazyStream(object):
258+    """
259+    The LazyStream wrapper allows one to pull and "unget" bytes from a stream.
260+
261+    Given a producer object (an iterator that yields bytestrings), the
262+    LazyStream object will support iteration, reading, and keeping a
263+    "look-back" variable in case you need to "unget" some bytes.
264+    """
265+    def __init__(self, producer, length=None):
266+        """
267+        Every LazyStream must have a producer when instantiated.
268+
269+        A producer is an iterable that returns a string each time it
270+        is called.
271+        """
272+        self._producer = producer
273+        self._empty = False
274+        self._leftover = ''
275+        self.length = length
276+        self.position = 0
277+        self._remaining = length
278+
279+    def tell(self):
280+        return self.position
281+
282+    def read(self, size=None):
283+        def parts():
284+            remaining = (size is not None and [size] or [self._remaining])[0]
285+            # do the whole thing in one shot if no limit was provided.
286+            if remaining is None:
287+                yield ''.join(self)
288+                return
289+
290+            # otherwise do some bookkeeping to return exactly enough
291+            # of the stream and stashing any extra content we get from
292+            # the producer
293+            while remaining != 0:
294+                assert remaining > 0, 'remaining bytes to read should never go negative'
295+
296+                chunk = self.next()
297+
298+                emitting = chunk[:remaining]
299+                self.unget(chunk[remaining:])
300+                remaining -= len(emitting)
301+                yield emitting
302+
303+        out = ''.join(parts())
304+        self.position += len(out)
305+        return out
306+
307+    def next(self):
308+        """
309+        Used when the exact number of bytes to read is unimportant.
310+
311+        This procedure just returns whatever is chunk is conveniently
312+        returned from the iterator instead. Useful to avoid
313+        unnecessary bookkeeping if performance is an issue.
314+        """
315+        if self._leftover:
316+            output = self._leftover
317+            self.position += len(output)
318+            self._leftover = ''
319+            return output
320+        else:
321+            output = self._producer.next()
322+            self.position += len(output)
323+            return output
324+
325+    def close(self):
326+        """
327+        Used to invalidate/disable this lazy stream.
328+
329+        Replaces the producer with an empty list. Any leftover bytes
330+        that have already been read will still be reported upon read()
331+        and/or next().
332+        """
333+        self._producer = []
334+
335+    def __iter__(self):
336+        return self
337+
338+    def unget(self, bytes):
339+        """
340+        Places bytes back onto the front of the lazy stream.
341+
342+        Future calls to read() will return those bytes first. The
343+        stream position and thus tell() will be rewound.
344+        """
345+        self.position -= len(bytes)
346+        self._leftover = ''.join([bytes, self._leftover])
347+
348+    def exhaust(self):
349+        """
350+        Exhausts the entire underlying stream.
351+
352+        Useful for skipping and advancing sections.
353+        """
354+        for thing in self:
355+            pass
356+
357+
358+class ChunkIter(object):
359+    """
360+    An iterable that will yield chunks of data.
361+    Given a file-like object as the constructor,
362+    this object will yield chunks of read operations
363+    from that object.
364+    """
365+    def __init__(self, flo, chunk_size=64 * 1024):
366+        self.flo = flo
367+        self.chunk_size = chunk_size
368+
369+    def next(self):
370+        try:
371+            data = self.flo.read(self.chunk_size)
372+        except InputStreamExhausted:
373+            raise StopIteration
374+        if data:
375+            return data
376+        else:
377+            raise StopIteration
378+
379+    def __iter__(self):
380+        return self
381+
382+
383+class LimitBytes(object):
384+    """ Limit bytes for a file object. """
385+    def __init__(self, fileobject, length):
386+        self._file = fileobject
387+        self.remaining = length
388+
389+    def read(self, num_bytes=None):
390+        """
391+        Read data from the underlying file.
392+        If you ask for too much or there isn't anything left,
393+        this will raise an InputStreamExhausted error.
394+        """
395+        if self.remaining <= 0:
396+            raise InputStreamExhausted()
397+        if num_bytes is None:
398+            num_bytes = self.remaining
399+        else:
400+            num_bytes = min(num_bytes, self.remaining)
401+        self.remaining -= num_bytes
402+        return self._file.read(num_bytes)
403+
404+    def exhaust(self):
405+        """
406+        Exhaust this file until all of the bytes it was limited by
407+        have been read.
408+        """
409+        while self.remaining > 0:
410+            num_bytes = min(self.remaining, 16384)
411+            __ = self._file.read(num_bytes)
412+            self.remaining -= num_bytes
413+
414+
415+class InterBoundaryIter(object):
416+    """
417+    A Producer that will iterate over boundaries.
418+    """
419+    def __init__(self, stream, boundary):
420+        self._stream = stream
421+        self._boundary = boundary
422+
423+    def __iter__(self):
424+        return self
425+
426+    def next(self):
427+        try:
428+            return LazyStream(BoundaryIter(self._stream, self._boundary))
429+        except InputStreamExhausted:
430+            raise StopIteration
431+
432+
433+class BoundaryIter(object):
434+    """
435+    A Producer that is sensitive to boundaries.
436+
437+    Will happily yield bytes until a boundary is found. Will yield the
438+    bytes before the boundary, throw away the boundary bytes
439+    themselves, and push the post-boundary bytes back on the stream.
440+
441+    The future calls to .next() after locating the boundary will raise
442+    a StopIteration exception.
443+    """
444+
445+    def __init__(self, stream, boundary):
446+        self._stream = stream
447+        self._boundary = boundary
448+        self._done = False
449+        # rollback an additional six bytes because the format is like
450+        # this: CRLF<boundary>[--CRLF]
451+        self._rollback = len(boundary) + 6
452+
453+        # Try to use mx fast string search if available. Otherwise
454+        # use Python find. Wrap the latter for consistency.
455+        unused_char = self._stream.read(1)
456+        if not unused_char:
457+            raise InputStreamExhausted
458+        self._stream.unget(unused_char)
459+        try:
460+            from mx.TextTools import FS
461+            self._fs = FS(boundary).find
462+        except ImportError:
463+            self._fs = lambda data: data.find(boundary)
464+
465+    def __iter__(self):
466+        return self
467+
468+    def next(self):
469+        if self._done:
470+            raise StopIteration
471+
472+        stream = self._stream
473+        rollback = self._rollback
474+
475+        bytes_read = 0
476+        chunks = []
477+        for bytes in stream:
478+            bytes_read += len(bytes)
479+            chunks.append(bytes)
480+            if bytes_read > rollback:
481+                break
482+            if not bytes:
483+                break
484+        else:
485+            self._done = True
486+
487+        if not chunks:
488+            raise StopIteration
489+
490+        chunk = ''.join(chunks)
491+        boundary = self._find_boundary(chunk, len(chunk) < self._rollback)
492+
493+        if boundary:
494+            end, next = boundary
495+            stream.unget(chunk[next:])
496+            self._done = True
497+            return chunk[:end]
498+        else:
499+            # make sure we dont treat a partial boundary (and
500+            # its separators) as data
501+            if not chunk[:-rollback]:# and len(chunk) >= (len(self._boundary) + 6):
502+                # There's nothing left, we should just return and mark as done.
503+                self._done = True
504+                return chunk
505+            else:
506+                stream.unget(chunk[-rollback:])
507+                return chunk[:-rollback]
508+
509+    def _find_boundary(self, data, eof = False):
510+        """
511+        Finds a multipart boundary in data.
512+
513+        Should no boundry exist in the data None is returned
514+        instead. Otherwise a tuple containing
515+        the indices of the following are returned:
516+
517+         * the end of current encapsulation
518+
519+         * the start of the next encapsulation
520+        """
521+        index = self._fs(data)
522+        if index < 0:
523+            return None
524+        else:
525+            end = index
526+            next = index + len(self._boundary)
527+            data_len = len(data) - 1
528+            # backup over CRLF
529+            if data[max(0,end-1)] == '\n':
530+                end -= 1
531+            if data[max(0,end-1)] == '\r':
532+                end -= 1
533+            # skip over --CRLF
534+            #if data[min(data_len,next)] == '-':
535+            #    next += 1
536+            #if data[min(data_len,next)] == '-':
537+            #    next += 1
538+            #if data[min(data_len,next)] == '\r':
539+            #    next += 1
540+            #if data[min(data_len,next)] == '\n':
541+            #    next += 1
542+            return end, next
543+
544+
545+def ParseBoundaryStream(stream, max_header_size):
546+        """
547+        Parses one and exactly one stream that encapsulates a boundary.
548+        """
549+        # Stream at beginning of header, look for end of header
550+        # and parse it if found. The header must fit within one
551+        # chunk.
552+        chunk = stream.read(max_header_size)
553+        # 'find' returns the top of these four bytes, so we'll
554+        # need to munch them later to prevent them from polluting
555+        # the payload.
556+        header_end = chunk.find('\r\n\r\n')
557+
558+        def _parse_header(line):
559+            main_value_pair, params = parse_header(line)
560+            try:
561+                name, value = main_value_pair.split(':', 1)
562+            except:
563+                raise ValueError("Invalid header: %r" % line)
564+            return name, (value, params)
565+
566+        if header_end == -1:
567+            # we find no header, so we just mark this fact and pass on
568+            # the stream verbatim
569+            stream.unget(chunk)
570+            return ('RAW', {}, stream)
571+
572+        header = chunk[:header_end]
573+
574+        # here we place any excess chunk back onto the stream, as
575+        # well as throwing away the CRLFCRLF bytes from above.
576+        stream.unget(chunk[header_end + 4:])
577+
578+        TYPE = 'RAW'
579+        outdict = {}
580+
581+        # Eliminate blank lines
582+        for line in header.split('\r\n'):
583+            # This terminology ("main value" and "dictionary of
584+            # parameters") is from the Python docs.
585+            try:
586+                name, (value, params) = _parse_header(line)
587+            except:
588+                continue
589+
590+            if name == 'content-disposition':
591+                TYPE = 'FIELD'
592+                if params.get('filename'):
593+                    TYPE = 'FILE'
594+
595+            outdict[name] = value, params
596+
597+        if TYPE == 'RAW':
598+            stream.unget(chunk)
599+
600+        return (TYPE, outdict, stream)
601+
602+
603+class Parser(object):
604+    def __init__(self, stream, boundary):
605+        self._stream = stream
606+        self._separator = '--' + boundary
607+
608+    def __iter__(self):
609+
610+        boundarystream = InterBoundaryIter(self._stream,
611+                                           self._separator)
612+
613+        for sub_stream in boundarystream:
614+            # Iterate over each part
615+            yield ParseBoundaryStream(sub_stream, 1024)
616+
617+def parse_header(line):
618+    """ Parse the header into a key-value. """
619+    plist = _parse_header_params(';' + line)
620+    key = plist.pop(0).lower()
621+    pdict = {}
622+    for p in plist:
623+        i = p.find('=')
624+        if i >= 0:
625+            name = p[:i].strip().lower()
626+            value = p[i+1:].strip()
627+            if len(value) >= 2 and value[0] == value[-1] == '"':
628+                value = value[1:-1]
629+                value = value.replace('\\\\', '\\').replace('\\"', '"')
630+            pdict[name] = value
631+    return key, pdict
632+
633+def _parse_header_params(s):
634+    plist = []
635+    while s[:1] == ';':
636+        s = s[1:]
637+        end = s.find(';')
638+        while end > 0 and s.count('"', 0, end) % 2:
639+            end = s.find(';', end + 1)
640+        if end < 0:
641+            end = len(s)
642+        f = s[:end]
643+        plist.append(f.strip())
644+        s = s[end:]
645+    return plist
646Index: django/http/__init__.py
647===================================================================
648--- django/http/__init__.py     (revision 7363)
649+++ django/http/__init__.py     (working copy)
650@@ -9,9 +9,9 @@
651 except ImportError:
652     from cgi import parse_qsl
653 
654-from django.utils.datastructures import MultiValueDict, FileDict
655+from django.utils.datastructures import MultiValueDict, ImmutableList
656 from django.utils.encoding import smart_str, iri_to_uri, force_unicode
657-
658+from django.http.multipartparser import MultiPartParser
659 from utils import *
660 
661 RESERVED_CHARS="!*'();:@&=+$,/?%#[]"
662@@ -25,6 +25,7 @@
663 
664     # The encoding used in GET/POST dicts. None means use default setting.
665     _encoding = None
666+    _upload_handlers = ()
667 
668     def __init__(self):
669         self.GET, self.POST, self.COOKIES, self.META, self.FILES = {}, {}, {}, {}, {}
670@@ -102,40 +103,41 @@
671 
672     encoding = property(_get_encoding, _set_encoding)
673 
674-def parse_file_upload(header_dict, post_data):
675-    """Returns a tuple of (POST QueryDict, FILES MultiValueDict)."""
676-    import email, email.Message
677-    from cgi import parse_header
678-    raw_message = '\r\n'.join(['%s:%s' % pair for pair in header_dict.items()])
679-    raw_message += '\r\n\r\n' + post_data
680-    msg = email.message_from_string(raw_message)
681-    POST = QueryDict('', mutable=True)
682-    FILES = MultiValueDict()
683-    for submessage in msg.get_payload():
684-        if submessage and isinstance(submessage, email.Message.Message):
685-            name_dict = parse_header(submessage['Content-Disposition'])[1]
686-            # name_dict is something like {'name': 'file', 'filename': 'test.txt'} for file uploads
687-            # or {'name': 'blah'} for POST fields
688-            # We assume all uploaded files have a 'filename' set.
689-            if 'filename' in name_dict:
690-                assert type([]) != type(submessage.get_payload()), "Nested MIME messages are not supported"
691-                if not name_dict['filename'].strip():
692-                    continue
693-                # IE submits the full path, so trim everything but the basename.
694-                # (We can't use os.path.basename because that uses the server's
695-                # directory separator, which may not be the same as the
696-                # client's one.)
697-                filename = name_dict['filename'][name_dict['filename'].rfind("\\")+1:]
698-                FILES.appendlist(name_dict['name'], FileDict({
699-                    'filename': filename,
700-                    'content-type': 'Content-Type' in submessage and submessage['Content-Type'] or None,
701-                    'content': submessage.get_payload(),
702-                }))
703-            else:
704-                POST.appendlist(name_dict['name'], submessage.get_payload())
705-    return POST, FILES
706+    def _initialize_handlers(self):
707+        from django.conf import settings
708+        from django.core.files.fileuploadhandler import load_handler
709+        handlers = []
710+        # We go through each handler in the settings variable
711+        # and instantiate the handler by calling HandlerClass(request).
712+        for handler in settings.FILE_UPLOAD_HANDLERS:
713+            handlers.append(load_handler(handler, self))
714+        self._upload_handlers = handlers
715 
716+    def _set_upload_handlers(self, upload_handlers):
717+        """
718+        Set the upload handler to the new handler given in the parameter.
719+        """
720+        if hasattr(self, '_files'):
721+            raise AttributeError("You cannot set the upload handlers after the upload has been processed.")
722+        self._upload_handlers = upload_handlers
723 
724+    def _get_upload_handlers(self):
725+        if not self._upload_handlers:
726+            # If thre are no upload handlers defined, initialize them from settings.
727+            self._initialize_handlers()
728+        return self._upload_handlers
729+
730+    upload_handlers = property(_get_upload_handlers, _set_upload_handlers)
731+
732+    def parse_file_upload(self, META, post_data):
733+        """Returns a tuple of (POST QueryDict, FILES MultiValueDict)."""
734+        self.upload_handlers = ImmutableList(self.upload_handlers,
735+                                             warning="You cannot alter the upload handlers after the upload has been processed.")
736+        parser = MultiPartParser(META, post_data, self.upload_handlers,
737+                                 self.encoding)
738+        return parser.parse()
739+
740+
741 class QueryDict(MultiValueDict):
742     """
743     A specialized MultiValueDict that takes a query string when initialized.
744Index: django/test/client.py
745===================================================================
746--- django/test/client.py       (revision 7363)
747+++ django/test/client.py       (working copy)
748@@ -18,6 +18,26 @@
749 BOUNDARY = 'BoUnDaRyStRiNg'
750 MULTIPART_CONTENT = 'multipart/form-data; boundary=%s' % BOUNDARY
751 
752+
753+class FakePayload(object):
754+    """
755+    A wrapper around StringIO that restricts what can be read,
756+    since data from the network can't be seeked and cannot
757+    be read outside of its content length (or else we hang).
758+    """
759+    def __init__(self, content):
760+        self.__content = StringIO(content)
761+        self.__len = len(content)
762+
763+    def read(self, num_bytes=None):
764+        if num_bytes is None:
765+            num_bytes = self.__len or 1
766+        assert self.__len >= num_bytes, "Cannot read more than the available bytes from the HTTP incoming data."
767+        content = self.__content.read(num_bytes)
768+        self.__len -= num_bytes
769+        return content
770+
771+
772 class ClientHandler(BaseHandler):
773     """
774     A HTTP Handler that can be used for testing purposes.
775@@ -230,7 +250,7 @@
776             'CONTENT_TYPE':   content_type,
777             'PATH_INFO':      urllib.unquote(path),
778             'REQUEST_METHOD': 'POST',
779-            'wsgi.input':     StringIO(post_data),
780+            'wsgi.input':     FakePayload(post_data),
781         }
782         r.update(extra)
783 
784Index: django/conf/global_settings.py
785===================================================================
786--- django/conf/global_settings.py      (revision 7363)
787+++ django/conf/global_settings.py      (working copy)
788@@ -224,6 +224,22 @@
789 # Example: "http://media.lawrence.com"
790 MEDIA_URL = ''
791 
792+# A tuple that enumerates the upload handlers
793+# in order.
794+FILE_UPLOAD_HANDLERS = (
795+    'django.core.files.fileuploadhandler.MemoryFileUploadHandler',
796+    'django.core.files.fileuploadhandler.TemporaryFileUploadHandler',
797+)
798+
799+# Number of bytes the length of the request can be before it is
800+# streamed to the file system instead of parsed entirely in memory.
801+FILE_UPLOAD_MAX_MEMORY_SIZE = 2621440
802+
803+# Directory to upload streamed files temporarily.
804+# A value of `None` means that it will use the default temporary
805+# directory for the server's operating system.
806+FILE_UPLOAD_TEMP_DIR = None
807+
808 # Default formatting for date objects. See all available format strings here:
809 # http://www.djangoproject.com/documentation/templates/#now
810 DATE_FORMAT = 'N j, Y'
811Index: django/db/models/base.py
812===================================================================
813--- django/db/models/base.py    (revision 7363)
814+++ django/db/models/base.py    (working copy)
815@@ -13,8 +13,10 @@
816 from django.utils.datastructures import SortedDict
817 from django.utils.functional import curry
818 from django.utils.encoding import smart_str, force_unicode, smart_unicode
819+from django.core.files.filemove import file_move_safe
820 from django.conf import settings
821 from itertools import izip
822+import warnings
823 import types
824 import sys
825 import os
826@@ -384,12 +386,24 @@
827     def _get_FIELD_size(self, field):
828         return os.path.getsize(self._get_FIELD_filename(field))
829 
830-    def _save_FIELD_file(self, field, filename, raw_contents, save=True):
831+    def _save_FIELD_file(self, field, filename, raw_field, save=True):
832         directory = field.get_directory_name()
833         try: # Create the date-based directory if it doesn't exist.
834             os.makedirs(os.path.join(settings.MEDIA_ROOT, directory))
835         except OSError: # Directory probably already exists.
836             pass
837+
838+        # Put the deprecation warning first since there are multiple
839+        # locations where we use the new and old interface.
840+        if isinstance(raw_field, dict):
841+            warnings.warn("The dictionary usage for files is deprecated. Use the new object interface instead.", DeprecationWarning)
842+
843+        if filename is None:
844+            try:
845+                filename = raw_field.file_name
846+            except AttributeError:
847+                filename = raw_field['filename']
848+
849         filename = field.get_filename(filename)
850 
851         # If the filename already exists, keep adding an underscore to the name of
852@@ -406,10 +420,26 @@
853         setattr(self, field.attname, filename)
854 
855         full_filename = self._get_FIELD_filename(field)
856-        fp = open(full_filename, 'wb')
857-        fp.write(raw_contents)
858-        fp.close()
859+        if hasattr(raw_field, 'temporary_file_path'):
860+            # This file has a file path that we can move.
861+            raw_field.close()
862+            file_move_safe(raw_field.temporary_file_path(), full_filename)
863+        else:
864+            from django.core.files import filelocks
865+            fp = open(full_filename, 'wb')
866+            # exclusive lock
867+            filelocks.lock(fp, filelocks.LOCK_EX)
868+            if hasattr(raw_field, 'chunk'):
869+                # This is a normal uploadedfile that we can stream.
870+                for chunk in raw_field.chunk(65535):
871+                    fp.write(chunk)
872+            else:
873+                # This is an old dictionary, use the old interface.
874+                fp.write(raw_field['content'])
875+            filelocks.unlock(fp)
876+            fp.close()
877 
878+
879         # Save the width and/or height, if applicable.
880         if isinstance(field, ImageField) and (field.width_field or field.height_field):
881             from django.utils.images import get_image_dimensions
882Index: django/db/models/fields/__init__.py
883===================================================================
884--- django/db/models/fields/__init__.py (revision 7363)
885+++ django/db/models/fields/__init__.py (working copy)
886@@ -785,7 +785,8 @@
887         setattr(cls, 'get_%s_filename' % self.name, curry(cls._get_FIELD_filename, field=self))
888         setattr(cls, 'get_%s_url' % self.name, curry(cls._get_FIELD_url, field=self))
889         setattr(cls, 'get_%s_size' % self.name, curry(cls._get_FIELD_size, field=self))
890-        setattr(cls, 'save_%s_file' % self.name, lambda instance, filename, raw_contents, save=True: instance._save_FIELD_file(self, filename, raw_contents, save))
891+        setattr(cls, 'save_%s_file' % self.name, lambda instance, filename, raw_field, save=True: instance._save_FIELD_file(self, filename, raw_field, save))
892+        setattr(cls, 'move_%s_file' % self.name, lambda instance, raw_field, save=True: instance._save_FIELD_file(self, None, raw_field, save))
893         dispatcher.connect(self.delete_file, signal=signals.post_delete, sender=cls)
894 
895     def delete_file(self, instance):
896@@ -808,10 +809,16 @@
897         if new_data.get(upload_field_name, False):
898             func = getattr(new_object, 'save_%s_file' % self.name)
899             if rel:
900-                func(new_data[upload_field_name][0]["filename"], new_data[upload_field_name][0]["content"], save)
901+                file = new_data[upload_field_name][0]
902             else:
903-                func(new_data[upload_field_name]["filename"], new_data[upload_field_name]["content"], save)
904+                file = new_data[upload_field_name]
905 
906+            try:
907+                file_name = file.file_name
908+            except AttributeError:
909+                file_name = file['filename']
910+            func(file_name, file, save)
911+
912     def get_directory_name(self):
913         return os.path.normpath(force_unicode(datetime.datetime.now().strftime(smart_str(self.upload_to))))
914 
915@@ -823,7 +830,7 @@
916     def save_form_data(self, instance, data):
917         from django.newforms.fields import UploadedFile
918         if data and isinstance(data, UploadedFile):
919-            getattr(instance, "save_%s_file" % self.name)(data.filename, data.content, save=False)
920+            getattr(instance, "save_%s_file" % self.name)(data.filename, data.data, save=False)
921 
922     def formfield(self, **kwargs):
923         defaults = {'form_class': forms.FileField}
924Index: django/oldforms/__init__.py
925===================================================================
926--- django/oldforms/__init__.py (revision 7363)
927+++ django/oldforms/__init__.py (working copy)
928@@ -680,18 +680,27 @@
929         self.field_name, self.is_required = field_name, is_required
930         self.validator_list = [self.isNonEmptyFile] + validator_list
931 
932-    def isNonEmptyFile(self, field_data, all_data):
933+    def isNonEmptyFile(self, new_data, all_data):
934+        if hasattr(new_data, 'upload_errors'):
935+            upload_errors = new_data.upload_errors()
936+            if upload_errors:
937+                raise validators.CriticalValidationError, upload_errors
938         try:
939-            content = field_data['content']
940-        except TypeError:
941-            raise validators.CriticalValidationError, ugettext("No file was submitted. Check the encoding type on the form.")
942-        if not content:
943+            file_size = new_data.file_size
944+        except AttributeError:
945+            file_size = len(new_data['content'])
946+        if not file_size:
947             raise validators.CriticalValidationError, ugettext("The submitted file is empty.")
948 
949     def render(self, data):
950         return mark_safe(u'<input type="file" id="%s" class="v%s" name="%s" />' % \
951             (self.get_id(), self.__class__.__name__, self.field_name))
952 
953+    def prepare(self, new_data):
954+        if hasattr(new_data, 'upload_errors'):
955+            upload_errors = new_data.upload_errors()
956+            new_data[self.field_name] = { '_file_upload_error': upload_errors }
957+
958     def html2python(data):
959         if data is None:
960             raise EmptyValue
961Index: django/core/handlers/wsgi.py
962===================================================================
963--- django/core/handlers/wsgi.py        (revision 7363)
964+++ django/core/handlers/wsgi.py        (working copy)
965@@ -112,9 +112,8 @@
966         # Populates self._post and self._files
967         if self.method == 'POST':
968             if self.environ.get('CONTENT_TYPE', '').startswith('multipart'):
969-                header_dict = dict([(k, v) for k, v in self.environ.items() if k.startswith('HTTP_')])
970-                header_dict['Content-Type'] = self.environ.get('CONTENT_TYPE', '')
971-                self._post, self._files = http.parse_file_upload(header_dict, self.raw_post_data)
972+                self._raw_post_data = ''
973+                self._post, self._files = self.parse_file_upload(self.META, self.environ['wsgi.input'])
974             else:
975                 self._post, self._files = http.QueryDict(self.raw_post_data, encoding=self._encoding), datastructures.MultiValueDict()
976         else:
977Index: django/core/handlers/modpython.py
978===================================================================
979--- django/core/handlers/modpython.py   (revision 7363)
980+++ django/core/handlers/modpython.py   (working copy)
981@@ -53,7 +53,8 @@
982     def _load_post_and_files(self):
983         "Populates self._post and self._files"
984         if 'content-type' in self._req.headers_in and self._req.headers_in['content-type'].startswith('multipart'):
985-            self._post, self._files = http.parse_file_upload(self._req.headers_in, self.raw_post_data)
986+            self._raw_post_data = ''
987+            self._post, self._files = self.parse_file_upload(self.META, self._req)
988         else:
989             self._post, self._files = http.QueryDict(self.raw_post_data, encoding=self._encoding), datastructures.MultiValueDict()
990 
991Index: django/core/files/filelocks.py
992===================================================================
993--- django/core/files/filelocks.py      (revision 0)
994+++ django/core/files/filelocks.py      (revision 0)
995@@ -0,0 +1,64 @@
996+"""
997+Locking portability based partially on example by
998+Jonathan Feignberg <jdf@pobox.com> in python cookbook.
999+
1000+Example Usage::
1001+
1002+    from django.core.files import filelocks
1003+
1004+    f = open('./file', 'wb')
1005+
1006+    filelocks.lock(f, filelocks.LOCK_EX)
1007+    f.write('Django')
1008+    f.close()
1009+"""
1010+
1011+__all__ = ('LOCK_EX','LOCK_SH','LOCK_NB','lock','unlock')
1012+
1013+system_type = None
1014+
1015+try:
1016+    import win32con
1017+    import win32file
1018+    import pywintypes
1019+    LOCK_EX = win32con.LOCKFILE_EXCLUSIVE_LOCK
1020+    LOCK_SH = 0
1021+    LOCK_NB = win32con.LOCKFILE_FAIL_IMMEDIATELY
1022+    __overlapped = pywintypes.OVERLAPPED()
1023+    system_type = 'nt'
1024+except (ImportError, AttributeError):
1025+    pass
1026+
1027+try:
1028+    import fcntl
1029+    LOCK_EX = fcntl.LOCK_EX
1030+    LOCK_SH = fcntl.LOCK_SH
1031+    LOCK_NB = fcntl.LOCK_NB
1032+    system_type = 'posix'
1033+except (ImportError, AttributeError):
1034+    pass
1035+
1036+if system_type == 'nt':
1037+    def lock(file, flags):
1038+               hfile = win32file._get_osfhandle(file.fileno())
1039+               win32file.LockFileEx(hfile, flags, 0, -0x10000, __overlapped)
1040+
1041+    def unlock(file):
1042+               hfile = win32file._get_osfhandle(file.fileno())
1043+               win32file.UnlockFileEx(hfile, 0, -0x10000, __overlapped)
1044+elif system_type == 'posix':
1045+    def lock(file, flags):
1046+               fcntl.flock(file.fileno(), flags)
1047+
1048+    def unlock(file):
1049+        fcntl.flock(file.fileno(), fcntl.LOCK_UN)
1050+else:
1051+    # File locking is not supported.
1052+    LOCK_EX = LOCK_SH = LOCK_NB = None
1053+       
1054+    # Dummy functions that don't do anything.
1055+    def lock(file, flags):
1056+               pass
1057+
1058+    def unlock(file):
1059+               pass
1060Index: django/core/files/uploadedfile.py
1061===================================================================
1062--- django/core/files/uploadedfile.py   (revision 0)
1063+++ django/core/files/uploadedfile.py   (revision 0)
1064@@ -0,0 +1,191 @@
1065+"""
1066+The uploaded file objects for Django.
1067+This contains the base UploadedFile and the TemporaryUploadedFile
1068+derived class.
1069+"""
1070+
1071+__all__ = ('UploadedFile', 'TemporaryUploadedFile', 'InMemoryUploadedFile')
1072+
1073+class UploadedFile(object):
1074+    """
1075+    The UploadedFile object behaves somewhat like a file
1076+    object and represents some data that the user submitted
1077+    and is stored in some form.
1078+    """
1079+    DEFAULT_CHUNK_SIZE = 64 * 2**10
1080+
1081+    def __init__(self):
1082+        self.file_size = None
1083+        self.file_name = None
1084+        self.content_type = None
1085+        self.charset = None
1086+        pass
1087+
1088+    def file_size(self):
1089+        return self.file_size
1090+
1091+    def chunk(self, chunk_size=None):
1092+        """
1093+        Read the file to generate chunks of chunk_size bytes.
1094+        """
1095+        if not chunk_size:
1096+            chunk_size = UploadedFile.DEFAULT_CHUNK_SIZE
1097+
1098+        if hasattr(self, 'seek'):
1099+            self.seek(0)
1100+        # Assume the pointer is at zero...
1101+        counter = self.file_size()
1102+
1103+        while counter > 0:
1104+            yield self.read(chunk_size)
1105+            counter -= chunk_size
1106+
1107+
1108+    def multiple_chunks(self, chunk_size=None):
1109+        """
1110+        Return True if you can expect multiple chunks, False otherwise.
1111+        Note: If a particular file representation is in memory, then
1112+              override this to return False.
1113+        """
1114+        if not chunk_size:
1115+            chunk_size = UploadedFile.DEFAULT_CHUNK_SIZE
1116+        return self.file_size() < chunk_size
1117+       
1118+
1119+    def read(self, num_bytes=None):
1120+        """
1121+        Read from the file in whatever representation it has.
1122+        """
1123+        raise NotImplementedError()
1124+
1125+    def open(self):
1126+        """
1127+        Open the file, if one needs to.
1128+        """
1129+        pass
1130+
1131+
1132+    def close(self):
1133+        """
1134+        Close the file, if one needs to.
1135+        """
1136+        pass
1137+
1138+    def __getitem__(self, key):
1139+        """
1140+        This maintains backwards compatibility.
1141+        """
1142+        import warnings
1143+        warnings.warn("The dictionary access of uploaded file objects is deprecated. Use the new object interface instead.", DeprecationWarning)
1144+        # Dictionary to translate labels
1145+        # for backwards compatbility.
1146+        # Should be removed at some point.
1147+        backwards_translate = {
1148+            'filename': 'file_name',
1149+            'content-type': 'content_type',
1150+            }
1151+
1152+        if key == 'content':
1153+            return self.read()
1154+        else:
1155+            return getattr(self, backwards_translate.get(key, key))
1156+
1157+    def __repr__(self):
1158+        """
1159+        This representation could be anything and can be overridden.
1160+        This is mostly done to make it look somewhat useful.
1161+        """
1162+        _dict = {
1163+            'file_name': self.file_name,
1164+            'content_type': self.content_type,
1165+            'content': '<omitted>',
1166+            }
1167+        return repr(_dict)
1168+
1169+
1170+class TemporaryUploadedFile(UploadedFile):
1171+    """
1172+    Upload a file to a temporary file.
1173+    """
1174+
1175+    def __init__(self, file, file_name, content_type, file_size, charset):
1176+        self.file = file
1177+        self.file_name = file_name
1178+        self.path = file.name
1179+        self.content_type = content_type
1180+        self.file_size = file_size
1181+        self.charset = charset
1182+        self.file.seek(0)
1183+
1184+    def temporary_file_path(self):
1185+        """
1186+        Return the full path of this file.
1187+        """
1188+        return self.path
1189+
1190+    def read(self, *args, **kwargs):
1191+        return self.file.read(*args, **kwargs)
1192+
1193+    def open(self):
1194+        """
1195+        Assume the person meant to seek.
1196+        """
1197+        self.seek(0)
1198+
1199+    def seek(self, *args, **kwargs):
1200+        self.file.seek(*args, **kwargs)
1201+
1202+
1203+class InMemoryUploadedFile(UploadedFile):
1204+    """
1205+    Upload a file into memory.
1206+    """
1207+    def __init__(self, file, field_name, file_name, content_type, charset):
1208+        self.file = file
1209+        self.field_name = field_name
1210+        self.file_name = file_name
1211+        self.content_type = content_type
1212+        self.charset = charset
1213+        self.file.seek(0)
1214+
1215+    def seek(self, *args, **kwargs):
1216+        self.file.seek(*args, **kwargs)
1217+
1218+    def open(self):
1219+        self.seek(0)
1220+
1221+    def read(self, *args, **kwargs):
1222+        return self.file.read(*args, **kwargs)
1223+
1224+    def chunk(self, chunk_size=None):
1225+        """
1226+        Return the entirety of the data regardless.
1227+        """
1228+        self.file.seek(0)
1229+        return self.read()
1230+
1231+    def multiple_chunks(self, chunk_size=None):
1232+        """
1233+        Since it's in memory, we'll never have multiple chunks.
1234+        """
1235+        return False
1236+
1237+
1238+class SimpleUploadedFile(InMemoryUploadedFile):
1239+    """
1240+    A simple representation of a file, which
1241+    just has content, size, and a name.
1242+    """
1243+    def __init__(self, name, content, content_type='text/plain'):
1244+        try:
1245+            from cStringIO import StringIO
1246+        except ImportError:
1247+            from StringIO import StringIO
1248+        self.file = StringIO(content or '')
1249+        self.file_name = name
1250+        self.field_name = None
1251+        self.file_size = len(content or '')
1252+        self.content_type = content_type
1253+        self.charset = None
1254+        self.file.seek(0)
1255+
1256Index: django/core/files/__init__.py
1257===================================================================
1258--- django/core/files/__init__.py       (revision 0)
1259+++ django/core/files/__init__.py       (revision 0)
1260@@ -0,0 +1 @@
1261+
1262Index: django/core/files/fileuploadhandler.py
1263===================================================================
1264--- django/core/files/fileuploadhandler.py      (revision 0)
1265+++ django/core/files/fileuploadhandler.py      (revision 0)
1266@@ -0,0 +1,225 @@
1267+""" A fileuploadhandler base and default subclass for handling file uploads.
1268+"""
1269+import os
1270+try:
1271+    from cStringIO import StringIO
1272+except ImportError:
1273+    from StringIO import StringIO
1274+
1275+from django.utils.encoding import force_unicode
1276+from django.utils.datastructures import MultiValueDict
1277+from django.core.exceptions import ImproperlyConfigured
1278+
1279+from django.core.files.uploadedfile import TemporaryUploadedFile, InMemoryUploadedFile
1280+
1281+__all__ = ('UploadFileException','StopUpload', 'SkipFile',
1282+           'FileUploadHandler', 'TemporaryFileUploadHandler',
1283+           'MemoryFileUploadHandler', 'load_handler')
1284+
1285+
1286+class UploadFileException(Exception):
1287+    """ Any error having to do with Uploading Files. """
1288+    pass
1289+
1290+class StopUpload(UploadFileException):
1291+    """ This exception is raised when an upload must abort. """
1292+    pass
1293+
1294+class SkipFile(UploadFileException):
1295+    """ This exception is raised when a file needs to be skipped. """
1296+    pass
1297+
1298+def load_handler(path, *args, **kwargs):
1299+    """
1300+    Given a path to a handler return the instantiation of that handler.
1301+    E.g.::
1302+        load_handler('django.core.files.fileuploadhandler.TemporaryFileUploadHandler', request)
1303+    will return a TemporaryFileUploadHandler object.
1304+    """
1305+    i = path.rfind('.')
1306+    module, attr = path[:i], path[i+1:]
1307+    try:
1308+        mod = __import__(module, {}, {}, [attr])
1309+    except ImportError, e:
1310+        raise ImproperlyConfigured, 'Error importing upload handler module %s: "%s"' % (module, e)
1311+    except ValueError, e:
1312+        raise ImproperlyConfigured, 'Error importing upload handler module Is FILE_UPLOAD_HANDLERS a correctly defined list or tuple?'
1313+    try:
1314+        cls = getattr(mod, attr)
1315+    except AttributeError:
1316+        raise ImproperlyConfigured, 'Module "%s" does not define a "%s" upload handler backend' % (module, attr)
1317+    return cls(*args, **kwargs)
1318+
1319+
1320+class FileUploadHandler(object):
1321+    """ FileUploadHandler will take data and handle file uploads
1322+    in a streamed fashion.
1323+    """
1324+    chunk_size = 64 * 2 ** 10 #: The default chunk size is 64 KB.
1325+
1326+    def __init__(self, request=None):
1327+        " Initialize some local variables. "
1328+        self.file_name = None
1329+        self.content_type = None
1330+        self.content_length = None
1331+        self.charset = None
1332+        self.request = request
1333+
1334+    def handle_raw_input(self, input_data, META, content_length, boundary, encoding=None):
1335+        """
1336+        Handle the raw input from the client.
1337+        Parameters:
1338+          *input_data* -- An object that supports reading via .read().
1339+          *content_length* -- The (integer) value of the Content-Length header from the client.
1340+          *boundary* -- The boundary from the Content-Type header. Be sure to prepend two '--'.
1341+        """
1342+        pass
1343+
1344+    def new_file(self, field_name, file_name, content_type, content_length, charset=None):
1345+        """
1346+        Signal that a new file has been started.
1347+       
1348+        Warning: Do not trust content_length, if you get it at all.
1349+        """
1350+        self.field_name = field_name
1351+        self.file_name = file_name
1352+        self.content_type = content_type
1353+        self.content_length = content_length
1354+        self.charset = charset
1355+
1356+    def receive_data_chunk(self, raw_data, start, stop):
1357+        """
1358+        Receive data from the streamed upload parser.
1359+        Start and stop are the positions in the file.
1360+        This equality should always be true::
1361+            len(raw_data) = stop - start
1362+        """
1363+        raise NotImplementedError()
1364+
1365+    def file_complete(self, file_size):
1366+        """
1367+        Signal that a file has completed.
1368+        File size corresponds to the actual size accumulated
1369+        by all the chunks.
1370+
1371+        This should return a valid UploadedFile object.
1372+        """
1373+        raise NotImplementedError()
1374+
1375+    def upload_complete(self):
1376+        """
1377+        Signal that the upload is complete.
1378+        Do any cleanup that is necessary for this handler.
1379+        """
1380+        pass
1381+
1382+
1383+
1384+class TemporaryFileUploadHandler(FileUploadHandler):
1385+    """
1386+    Upload the streaming data into a temporary file.
1387+    """
1388+    def __init__(self, *args, **kwargs):
1389+        """ Import settings for later. """
1390+        super(TemporaryFileUploadHandler, self).__init__(*args, **kwargs)
1391+        global settings
1392+        from django.conf import settings
1393+
1394+    def new_file(self, file_name, *args, **kwargs):
1395+        """
1396+        Create the file object to append to as data is coming in.
1397+        """
1398+        super(TemporaryFileUploadHandler, self).new_file(file_name, *args, **kwargs)
1399+        self.file = TemporaryFile(settings.FILE_UPLOAD_TEMP_DIR)
1400+        self.write = self.file.write
1401+
1402+    def receive_data_chunk(self, raw_data, start, stop):
1403+        """
1404+        Once we get the data, we will save it to our file.
1405+        """
1406+        self.write(raw_data)
1407+
1408+    def file_complete(self, file_size):
1409+        """
1410+        Signal that a file has completed.
1411+        File size corresponds to the actual size accumulated
1412+        by all the chunks.
1413+
1414+        This should return a valid UploadedFile object.
1415+        """
1416+        self.file.seek(0)
1417+        return TemporaryUploadedFile(self.file, self.file_name,
1418+                                     self.content_type, file_size,
1419+                                     self.charset)
1420+
1421+
1422+class MemoryFileUploadHandler(FileUploadHandler):
1423+    """
1424+    The MemoryFileUploadHandler will place the data directly into memory.
1425+    """
1426+
1427+    def handle_raw_input(self, input_data, META, content_length, boundary, encoding=None):
1428+        """
1429+        Use the content_length to signal whether or not this handler should be in use.
1430+        """
1431+        global settings
1432+        from django.conf import settings
1433+
1434+        # If the the post is too large, we cannot use the Memory handler.
1435+        if content_length > settings.FILE_UPLOAD_MAX_MEMORY_SIZE:
1436+            self.activated = False
1437+        else:
1438+            self.activated = True
1439+
1440+    def new_file(self, *args, **kwargs):
1441+        super(MemoryFileUploadHandler, self).new_file(*args, **kwargs)
1442+        if self.activated:
1443+            self.file = StringIO()
1444+            return "Stop"
1445+
1446+    def receive_data_chunk(self, raw_data, start, stop):
1447+        """
1448+        Add the data to the StringIO file.
1449+        """
1450+        if self.activated:
1451+            self.file.write(raw_data)
1452+        else:
1453+            return raw_data
1454+
1455+    def file_complete(self, file_size):
1456+        """
1457+        Return a file object if we're activated.
1458+        """
1459+        if not self.activated:
1460+            return
1461+
1462+        return InMemoryUploadedFile(self.file, self.field_name, self.file_name,
1463+                                    self.content_type, self.charset)
1464+
1465+
1466+class TemporaryFile(object):
1467+    """
1468+    A temporary file that tries to delete itself when garbage collected.
1469+    """
1470+    def __init__(self, dir):
1471+        import tempfile
1472+        if not dir:
1473+            dir = tempfile.gettempdir()
1474+        try:
1475+            (fd, name) = tempfile.mkstemp(suffix='.upload', dir=dir)
1476+            self.file = os.fdopen(fd, 'w+b')
1477+        except (OSError, IOError):
1478+            raise OSError, "Could not create temporary file for uploading, have you set settings.FILE_UPLOAD_TEMP_DIR correctly?"
1479+        self.name = name
1480+
1481+    def __getattr__(self, name):
1482+        a = getattr(self.__dict__['file'], name)
1483+        if type(a) != type(0):
1484+            setattr(self, name, a)
1485+        return a
1486+
1487+    def __del__(self):
1488+        try:
1489+            os.unlink(self.name)
1490+        except OSError:
1491+            pass
1492Index: django/core/files/filemove.py
1493===================================================================
1494--- django/core/files/filemove.py       (revision 0)
1495+++ django/core/files/filemove.py       (revision 0)
1496@@ -0,0 +1,52 @@
1497+import os
1498+
1499+__all__ = ('file_move_safe',)
1500+
1501+try:
1502+    import shutil
1503+    file_move = shutil.move
1504+except ImportError:
1505+    file_move = os.rename
1506+
1507+def file_move_safe(old_file_name, new_file_name, chunk_size = 1024*64, allow_overwrite=False):
1508+    """
1509+    Moves a file from one location to another in the safest way possible.
1510+   
1511+    First, it tries using shutils.move, which is OS-dependent but doesn't
1512+    break with change of filesystems. Then it tries os.rename, which will
1513+    break if it encounters a change in filesystems. Lastly, it streams
1514+    it manually from one file to another in python.
1515+
1516+    Without ``allow_overwrite``, if the destination file exists, the
1517+    file will raise an IOError.
1518+    """
1519+
1520+    from django.core.files import filelocks
1521+
1522+    if old_file_name == new_file_name:
1523+        # No file moving takes place.
1524+        return
1525+
1526+    if not allow_overwrite and os.path.exists(new_file_name):
1527+        raise IOError, "Django does not allow overwriting files."
1528+
1529+    try:
1530+        file_move(old_file_name, new_file_name)
1531+        return
1532+    except OSError: # moving to another filesystem
1533+        pass
1534+
1535+    new_file = open(new_file_name, 'wb')
1536+    # exclusive lock
1537+    filelocks.lock(new_file, filelocks.LOCK_EX)
1538+    old_file = open(old_file_name, 'rb')
1539+    current_chunk = None
1540+
1541+    while current_chunk != '':
1542+        current_chunk = old_file.read(chunk_size)
1543+        new_file.write(current_chunk)
1544+
1545+    new_file.close()
1546+    old_file.close()
1547+
1548+    os.remove(old_file_name)
1549Index: django/utils/datastructures.py
1550===================================================================
1551--- django/utils/datastructures.py      (revision 7363)
1552+++ django/utils/datastructures.py      (working copy)
1553@@ -332,14 +332,34 @@
1554             except TypeError: # Special-case if current isn't a dict.
1555                 current = {bits[-1]: v}
1556 
1557-class FileDict(dict):
1558+class ImmutableList(tuple):
1559     """
1560-    A dictionary used to hold uploaded file contents. The only special feature
1561-    here is that repr() of this object won't dump the entire contents of the
1562-    file to the output. A handy safeguard for a large file upload.
1563+    A tuple-like object that raises useful
1564+    errors when it is asked to mutate.
1565+    Example::
1566+        a = ImmutableList(range(5), warning=AttributeError("You cannot mutate this."))
1567+        a[3] = '4'
1568+        (Raises the AttributeError)
1569     """
1570-    def __repr__(self):
1571-        if 'content' in self:
1572-            d = dict(self, content='<omitted>')
1573-            return dict.__repr__(d)
1574-        return dict.__repr__(self)
1575+
1576+    def __new__(cls, *args, **kwargs):
1577+        if 'warning' in kwargs:
1578+            warning = kwargs['warning']
1579+            del kwargs['warning']
1580+        else:
1581+            warning = 'ImmutableList object is immutable.'
1582+        self = tuple.__new__(cls, *args, **kwargs)
1583+        self.warning = warning
1584+        return self
1585+
1586+    def complain(self, *wargs, **kwargs):
1587+        if isinstance(self.warning, Exception):
1588+            raise self.warning
1589+        else:
1590+            raise AttributeError, self.warning
1591+
1592+    # All list mutation functions become complain.
1593+    __delitem__ = __delslice__ = __iadd__ = __imul__ = complain
1594+    __setitem__ = __setslice__ = complain
1595+    append = extend = insert = pop = remove = complain
1596+    sort = reverse = complain
1597Index: django/utils/text.py
1598===================================================================
1599--- django/utils/text.py        (revision 7363)
1600+++ django/utils/text.py        (working copy)
1601@@ -3,6 +3,7 @@
1602 from django.utils.encoding import force_unicode
1603 from django.utils.functional import allow_lazy
1604 from django.utils.translation import ugettext_lazy
1605+from htmlentitydefs import name2codepoint
1606 
1607 # Capitalizes the first letter of a string.
1608 capfirst = lambda x: x and force_unicode(x)[0].upper() + force_unicode(x)[1:]
1609@@ -218,3 +219,26 @@
1610             yield bit
1611 smart_split = allow_lazy(smart_split, unicode)
1612 
1613+def _replace_entity(match):
1614+     text = match.group(1)
1615+     if text[0] == u'#':
1616+         text = text[1:]
1617+         try:
1618+             if text[0] in u'xX':
1619+                 c = int(text[1:], 16)
1620+             else:
1621+                 c = int(text)
1622+             return unichr(c)
1623+         except ValueError:
1624+             return match.group(0)
1625+     else:
1626+         try:
1627+             return unichr(name2codepoint[text])
1628+         except (ValueError, KeyError):
1629+             return match.group(0)
1630+
1631+_entity_re = re.compile(r"&(#?[xX]?(?:[0-9a-fA-F]+|\w{1,8}));")
1632+
1633+def unescape_entities(text):
1634+     return _entity_re.sub(_replace_entity, text)
1635+unescape_entities = allow_lazy(unescape_entities, unicode)
1636Index: django/newforms/fields.py
1637===================================================================
1638--- django/newforms/fields.py   (revision 7363)
1639+++ django/newforms/fields.py   (working copy)
1640@@ -4,6 +4,7 @@
1641 
1642 import copy
1643 import datetime
1644+import warnings
1645 import os
1646 import re
1647 import time
1648@@ -416,9 +417,9 @@
1649 
1650 class UploadedFile(StrAndUnicode):
1651     "A wrapper for files uploaded in a FileField"
1652-    def __init__(self, filename, content):
1653+    def __init__(self, filename, data):
1654         self.filename = filename
1655-        self.content = content
1656+        self.data = data
1657 
1658     def __unicode__(self):
1659         """
1660@@ -444,16 +445,28 @@
1661             return None
1662         elif not data and initial:
1663             return initial
1664+
1665+        if isinstance(data, dict):
1666+            # We warn once, then support both ways below.
1667+            warnings.warn("The dictionary usage for files is deprecated. Use the new object interface instead.", DeprecationWarning)
1668+
1669         try:
1670-            f = UploadedFile(data['filename'], data['content'])
1671-        except TypeError:
1672+            file_name = data.file_name
1673+            file_size = data.file_size
1674+        except AttributeError:
1675+            try:
1676+                file_name = data.get('filename')
1677+                file_size = bool(data['content'])
1678+            except (AttributeError, KeyError):
1679+                raise ValidationError(self.error_messages['invalid'])
1680+
1681+        if not file_name:
1682             raise ValidationError(self.error_messages['invalid'])
1683-        except KeyError:
1684-            raise ValidationError(self.error_messages['missing'])
1685-        if not f.content:
1686+        if not file_size:
1687             raise ValidationError(self.error_messages['empty'])
1688-        return f
1689 
1690+        return UploadedFile(file_name, data)
1691+
1692 class ImageField(FileField):
1693     default_error_messages = {
1694         'invalid_image': _(u"Upload a valid image. The file you uploaded was either not an image or a corrupted image."),
1695@@ -470,15 +483,28 @@
1696         elif not data and initial:
1697             return initial
1698         from PIL import Image
1699-        from cStringIO import StringIO
1700+
1701+        # We need to get the file, it either has a path
1702+        # or we have to read it all into memory...
1703+        if hasattr(data, 'temporary_file_path'):
1704+            file = data.temporary_file_path()
1705+        else:
1706+            try:
1707+                from cStringIO import StringIO
1708+            except ImportError:
1709+                from StringIO import StringIO
1710+            if hasattr(data, 'read'):
1711+                file = StringIO(data.read())
1712+            else:
1713+                file = StringIO(data['content'])
1714         try:
1715             # load() is the only method that can spot a truncated JPEG,
1716             #  but it cannot be called sanely after verify()
1717-            trial_image = Image.open(StringIO(f.content))
1718+            trial_image = Image.open(file)
1719             trial_image.load()
1720             # verify() is the only method that can spot a corrupt PNG,
1721             #  but it must be called immediately after the constructor
1722-            trial_image = Image.open(StringIO(f.content))
1723+            trial_image = Image.open(file)
1724             trial_image.verify()
1725         except Exception: # Python Imaging Library doesn't recognize it as an image
1726             raise ValidationError(self.error_messages['invalid_image'])
1727Index: tests/modeltests/model_forms/models.py
1728===================================================================
1729--- tests/modeltests/model_forms/models.py      (revision 7363)
1730+++ tests/modeltests/model_forms/models.py      (working copy)
1731@@ -75,6 +75,9 @@
1732 __test__ = {'API_TESTS': """
1733 >>> from django import newforms as forms
1734 >>> from django.newforms.models import ModelForm
1735+>>> from django.core.files.uploadedfile import SimpleUploadedFile
1736+>>> from warnings import filterwarnings
1737+>>> filterwarnings("ignore")
1738 
1739 The bare bones, absolutely nothing custom, basic case.
1740 
1741@@ -792,6 +795,17 @@
1742 
1743 # Upload a file and ensure it all works as expected.
1744 
1745+>>> f = TextFileForm(data={'description': u'Assistance'}, files={'file': SimpleUploadedFile('test1.txt', 'hello world')})
1746+>>> f.is_valid()
1747+True
1748+>>> type(f.cleaned_data['file'])
1749+<class 'django.newforms.fields.UploadedFile'>
1750+>>> instance = f.save()
1751+>>> instance.file
1752+u'...test1.txt'
1753+
1754+>>> os.unlink(instance.get_file_filename())
1755+
1756 >>> f = TextFileForm(data={'description': u'Assistance'}, files={'file': {'filename': 'test1.txt', 'content': 'hello world'}})
1757 >>> f.is_valid()
1758 True
1759@@ -814,18 +828,30 @@
1760 u'...test1.txt'
1761 
1762 # Delete the current file since this is not done by Django.
1763-
1764 >>> os.unlink(instance.get_file_filename())
1765 
1766 # Override the file by uploading a new one.
1767 
1768->>> f = TextFileForm(data={'description': u'Assistance'}, files={'file': {'filename': 'test2.txt', 'content': 'hello world'}}, instance=instance)
1769+>>> f = TextFileForm(data={'description': u'Assistance'}, files={'file': SimpleUploadedFile('test2.txt', 'hello world')}, instance=instance)
1770 >>> f.is_valid()
1771 True
1772 >>> instance = f.save()
1773 >>> instance.file
1774 u'...test2.txt'
1775 
1776+# Delete the current file since this is not done by Django.
1777+>>> os.unlink(instance.get_file_filename())
1778+
1779+>>> f = TextFileForm(data={'description': u'Assistance'}, files={'file': {'filename': 'test2.txt', 'content': 'hello world'}})
1780+>>> f.is_valid()
1781+True
1782+>>> instance = f.save()
1783+>>> instance.file
1784+u'...test2.txt'
1785+
1786+# Delete the current file since this is not done by Django.
1787+>>> os.unlink(instance.get_file_filename())
1788+
1789 >>> instance.delete()
1790 
1791 # Test the non-required FileField
1792@@ -838,14 +864,28 @@
1793 >>> instance.file
1794 ''
1795 
1796->>> f = TextFileForm(data={'description': u'Assistance'}, files={'file': {'filename': 'test3.txt', 'content': 'hello world'}}, instance=instance)
1797+>>> f = TextFileForm(data={'description': u'Assistance'}, files={'file': SimpleUploadedFile('test3.txt', 'hello world')}, instance=instance)
1798 >>> f.is_valid()
1799 True
1800 >>> instance = f.save()
1801 >>> instance.file
1802 u'...test3.txt'
1803+
1804+# Delete the current file since this is not done by Django.
1805+>>> os.unlink(instance.get_file_filename())
1806 >>> instance.delete()
1807 
1808+>>> f = TextFileForm(data={'description': u'Assistance'}, files={'file': {'filename': 'test3.txt', 'content': 'hello world'}})
1809+>>> f.is_valid()
1810+True
1811+>>> instance = f.save()
1812+>>> instance.file
1813+u'...test3.txt'
1814+
1815+# Delete the current file since this is not done by Django.
1816+>>> os.unlink(instance.get_file_filename())
1817+>>> instance.delete()
1818+
1819 # ImageField ###################################################################
1820 
1821 # ImageField and FileField are nearly identical, but they differ slighty when
1822@@ -858,6 +898,18 @@
1823 
1824 >>> image_data = open(os.path.join(os.path.dirname(__file__), "test.png")).read()
1825 
1826+>>> f = ImageFileForm(data={'description': u'An image'}, files={'image': SimpleUploadedFile('test.png', image_data)})
1827+>>> f.is_valid()
1828+True
1829+>>> type(f.cleaned_data['image'])
1830+<class 'django.newforms.fields.UploadedFile'>
1831+>>> instance = f.save()
1832+>>> instance.image
1833+u'...test.png'
1834+
1835+# Delete the current file since this is not done by Django.
1836+>>> os.unlink(instance.get_image_filename())
1837+
1838 >>> f = ImageFileForm(data={'description': u'An image'}, files={'image': {'filename': 'test.png', 'content': image_data}})
1839 >>> f.is_valid()
1840 True
1841@@ -885,15 +937,28 @@
1842 
1843 # Override the file by uploading a new one.
1844 
1845->>> f = ImageFileForm(data={'description': u'Changed it'}, files={'image': {'filename': 'test2.png', 'content': image_data}}, instance=instance)
1846+>>> f = ImageFileForm(data={'description': u'Changed it'}, files={'image': SimpleUploadedFile('test2.png', image_data)}, instance=instance)
1847 >>> f.is_valid()
1848 True
1849 >>> instance = f.save()
1850 >>> instance.image
1851 u'...test2.png'
1852 
1853+# Delete the current file since this is not done by Django.
1854+>>> os.unlink(instance.get_image_filename())
1855 >>> instance.delete()
1856 
1857+>>> f = ImageFileForm(data={'description': u'Changed it'}, files={'image': {'filename': 'test2.png', 'content': image_data}})
1858+>>> f.is_valid()
1859+True
1860+>>> instance = f.save()
1861+>>> instance.image
1862+u'...test2.png'
1863+
1864+# Delete the current file since this is not done by Django.
1865+>>> os.unlink(instance.get_image_filename())
1866+>>> instance.delete()
1867+
1868 # Test the non-required ImageField
1869 
1870 >>> f = ImageFileForm(data={'description': u'Test'})
1871@@ -904,12 +969,23 @@
1872 >>> instance.image
1873 ''
1874 
1875->>> f = ImageFileForm(data={'description': u'And a final one'}, files={'image': {'filename': 'test3.png', 'content': image_data}}, instance=instance)
1876+>>> f = ImageFileForm(data={'description': u'And a final one'}, files={'image': SimpleUploadedFile('test3.png', image_data)}, instance=instance)
1877 >>> f.is_valid()
1878 True
1879 >>> instance = f.save()
1880 >>> instance.image
1881 u'...test3.png'
1882+
1883+# Delete the current file since this is not done by Django.
1884+>>> os.unlink(instance.get_image_filename())
1885 >>> instance.delete()
1886 
1887+>>> f = ImageFileForm(data={'description': u'And a final one'}, files={'image': {'filename': 'test3.png', 'content': image_data}})
1888+>>> f.is_valid()
1889+True
1890+>>> instance = f.save()
1891+>>> instance.image
1892+u'...test3.png'
1893+>>> instance.delete()
1894+
1895 """}
1896Index: tests/regressiontests/bug639/tests.py
1897===================================================================
1898--- tests/regressiontests/bug639/tests.py       (revision 7363)
1899+++ tests/regressiontests/bug639/tests.py       (working copy)
1900@@ -9,6 +9,7 @@
1901 from regressiontests.bug639.models import Photo
1902 from django.http import QueryDict
1903 from django.utils.datastructures import MultiValueDict
1904+from django.core.files.uploadedfile import SimpleUploadedFile
1905 
1906 class Bug639Test(unittest.TestCase):
1907         
1908@@ -21,12 +22,13 @@
1909         
1910         # Fake a request query dict with the file
1911         qd = QueryDict("title=Testing&image=", mutable=True)
1912-        qd["image_file"] = {
1913-            "filename" : "test.jpg",
1914-            "content-type" : "image/jpeg",
1915-            "content" : img
1916-        }
1917-       
1918+        #qd["image_file"] = {
1919+        #    "filename" : "test.jpg",
1920+        #    "content-type" : "image/jpeg",
1921+        #    "content" : img
1922+        #    }
1923+        qd["image_file"] = SimpleUploadedFile('test.jpg', img, 'image/jpeg')
1924+
1925         manip = Photo.AddManipulator()
1926         manip.do_html2python(qd)
1927         p = manip.save(qd)
1928@@ -39,4 +41,4 @@
1929         Make sure to delete the "uploaded" file to avoid clogging /tmp.
1930         """
1931         p = Photo.objects.get()
1932-        os.unlink(p.get_image_filename())
1933\ No newline at end of file
1934+        os.unlink(p.get_image_filename())
1935Index: tests/regressiontests/forms/error_messages.py
1936===================================================================
1937--- tests/regressiontests/forms/error_messages.py       (revision 7363)
1938+++ tests/regressiontests/forms/error_messages.py       (working copy)
1939@@ -1,6 +1,7 @@
1940 # -*- coding: utf-8 -*-
1941 tests = r"""
1942 >>> from django.newforms import *
1943+>>> from django.core.files.uploadedfile import SimpleUploadedFile
1944 
1945 # CharField ###################################################################
1946 
1947@@ -214,11 +215,11 @@
1948 Traceback (most recent call last):
1949 ...
1950 ValidationError: [u'INVALID']
1951->>> f.clean({})
1952+>>> f.clean(SimpleUploadedFile('name', None))
1953 Traceback (most recent call last):
1954 ...
1955-ValidationError: [u'MISSING']
1956->>> f.clean({'filename': 'name', 'content':''})
1957+ValidationError: [u'EMPTY FILE']
1958+>>> f.clean(SimpleUploadedFile('name', ''))
1959 Traceback (most recent call last):
1960 ...
1961 ValidationError: [u'EMPTY FILE']
1962Index: tests/regressiontests/forms/tests.py
1963===================================================================
1964--- tests/regressiontests/forms/tests.py        (revision 7363)
1965+++ tests/regressiontests/forms/tests.py        (working copy)
1966@@ -26,6 +26,8 @@
1967 from regressions import tests as regression_tests
1968 from util import tests as util_tests
1969 from widgets import tests as widgets_tests
1970+from warnings import filterwarnings
1971+filterwarnings("ignore")
1972 
1973 __test__ = {
1974     'extra_tests': extra_tests,
1975Index: tests/regressiontests/forms/fields.py
1976===================================================================
1977--- tests/regressiontests/forms/fields.py       (revision 7363)
1978+++ tests/regressiontests/forms/fields.py       (working copy)
1979@@ -2,6 +2,7 @@
1980 tests = r"""
1981 >>> from django.newforms import *
1982 >>> from django.newforms.widgets import RadioFieldRenderer
1983+>>> from django.core.files.uploadedfile import SimpleUploadedFile
1984 >>> import datetime
1985 >>> import time
1986 >>> import re
1987@@ -773,12 +774,12 @@
1988 >>> f.clean({})
1989 Traceback (most recent call last):
1990 ...
1991-ValidationError: [u'No file was submitted.']
1992+ValidationError: [u'No file was submitted. Check the encoding type on the form.']
1993 
1994 >>> f.clean({}, '')
1995 Traceback (most recent call last):
1996 ...
1997-ValidationError: [u'No file was submitted.']
1998+ValidationError: [u'No file was submitted. Check the encoding type on the form.']
1999 
2000 >>> f.clean({}, 'files/test3.pdf')
2001 'files/test3.pdf'
2002@@ -788,20 +789,20 @@
2003 ...
2004 ValidationError: [u'No file was submitted. Check the encoding type on the form.']
2005 
2006->>> f.clean({'filename': 'name', 'content': None})
2007+>>> f.clean(SimpleUploadedFile('name', None))
2008 Traceback (most recent call last):
2009 ...
2010 ValidationError: [u'The submitted file is empty.']
2011 
2012->>> f.clean({'filename': 'name', 'content': ''})
2013+>>> f.clean(SimpleUploadedFile('name', ''))
2014 Traceback (most recent call last):
2015 ...
2016 ValidationError: [u'The submitted file is empty.']
2017 
2018->>> type(f.clean({'filename': 'name', 'content': 'Some File Content'}))
2019+>>> type(f.clean(SimpleUploadedFile('name', 'Some File Content')))
2020 <class 'django.newforms.fields.UploadedFile'>
2021 
2022->>> type(f.clean({'filename': 'name', 'content': 'Some File Content'}, 'files/test4.pdf'))
2023+>>> type(f.clean(SimpleUploadedFile('name', 'Some File Content'), 'files/test4.pdf'))
2024 <class 'django.newforms.fields.UploadedFile'>
2025 
2026 # URLField ##################################################################
2027Index: tests/regressiontests/forms/forms.py
2028===================================================================
2029--- tests/regressiontests/forms/forms.py        (revision 7363)
2030+++ tests/regressiontests/forms/forms.py        (working copy)
2031@@ -1,6 +1,7 @@
2032 # -*- coding: utf-8 -*-
2033 tests = r"""
2034 >>> from django.newforms import *
2035+>>> from django.core.files.uploadedfile import SimpleUploadedFile
2036 >>> import datetime
2037 >>> import time
2038 >>> import re
2039@@ -1465,7 +1466,7 @@
2040 >>> print f
2041 <tr><th>File1:</th><td><ul class="errorlist"><li>This field is required.</li></ul><input type="file" name="file1" /></td></tr>
2042 
2043->>> f = FileForm(data={}, files={'file1': {'filename': 'name', 'content':''}}, auto_id=False)
2044+>>> f = FileForm(data={}, files={'file1': SimpleUploadedFile('name', '')}, auto_id=False)
2045 >>> print f
2046 <tr><th>File1:</th><td><ul class="errorlist"><li>The submitted file is empty.</li></ul><input type="file" name="file1" /></td></tr>
2047 
2048@@ -1473,7 +1474,7 @@
2049 >>> print f
2050 <tr><th>File1:</th><td><ul class="errorlist"><li>No file was submitted. Check the encoding type on the form.</li></ul><input type="file" name="file1" /></td></tr>
2051 
2052->>> f = FileForm(data={}, files={'file1': {'filename': 'name', 'content':'some content'}}, auto_id=False)
2053+>>> f = FileForm(data={}, files={'file1': SimpleUploadedFile('name', 'some content')}, auto_id=False)
2054 >>> print f
2055 <tr><th>File1:</th><td><input type="file" name="file1" /></td></tr>
2056 >>> f.is_valid()
2057Index: tests/regressiontests/test_client_regress/views.py
2058===================================================================
2059--- tests/regressiontests/test_client_regress/views.py  (revision 7363)
2060+++ tests/regressiontests/test_client_regress/views.py  (working copy)
2061@@ -1,5 +1,6 @@
2062 from django.contrib.auth.decorators import login_required
2063 from django.http import HttpResponse, HttpResponseRedirect, HttpResponseServerError
2064+import sha
2065 
2066 def no_template_view(request):
2067     "A simple view that expects a GET request, and returns a rendered template"
2068@@ -10,13 +11,41 @@
2069     Check that a file upload can be updated into the POST dictionary without
2070     going pear-shaped.
2071     """
2072+    from django.core.files.uploadedfile import UploadedFile
2073     form_data = request.POST.copy()
2074     form_data.update(request.FILES)
2075-    if isinstance(form_data['file_field'], dict) and isinstance(form_data['name'], unicode):
2076+    if isinstance(form_data.get('file_field'), UploadedFile) and isinstance(form_data['name'], unicode):
2077         return HttpResponse('')
2078     else:
2079         return HttpResponseServerError()
2080 
2081+def file_upload_view_verify(request):
2082+    """
2083+    Use the sha digest hash to verify the uploaded contents.
2084+    """
2085+    from django.core.files.uploadedfile import UploadedFile
2086+    form_data = request.POST.copy()
2087+    form_data.update(request.FILES)
2088+
2089+    # Check to see if unicode names worked out.
2090+    if not request.FILES['file_unicode'].file_name.endswith(u'test_\u4e2d\u6587_Orl\xe9ans.jpg'):
2091+        return HttpResponseServerError()
2092+
2093+    for key, value in form_data.items():
2094+        if key.endswith('_hash'):
2095+            continue
2096+        if key + '_hash' not in form_data:
2097+            continue
2098+        submitted_hash = form_data[key + '_hash']
2099+        if isinstance(value, UploadedFile):
2100+            new_hash = sha.new(value.read()).hexdigest()
2101+        else:
2102+            new_hash = sha.new(value).hexdigest()
2103+        if new_hash != submitted_hash:
2104+            return HttpResponseServerError()
2105+
2106+    return HttpResponse('')
2107+
2108 def get_view(request):
2109     "A simple login protected view"
2110     return HttpResponse("Hello world")
2111@@ -37,4 +66,4 @@
2112 def login_protected_redirect_view(request):
2113     "A view that redirects all requests to the GET view"
2114     return HttpResponseRedirect('/test_client_regress/get_view/')
2115-login_protected_redirect_view = login_required(login_protected_redirect_view)
2116\ No newline at end of file
2117+login_protected_redirect_view = login_required(login_protected_redirect_view)
2118Index: tests/regressiontests/test_client_regress/models.py
2119===================================================================
2120--- tests/regressiontests/test_client_regress/models.py (revision 7363)
2121+++ tests/regressiontests/test_client_regress/models.py (working copy)
2122@@ -5,6 +5,7 @@
2123 from django.test import Client, TestCase
2124 from django.core.urlresolvers import reverse
2125 import os
2126+import sha
2127 
2128 class AssertContainsTests(TestCase):
2129     def test_contains(self):
2130@@ -243,6 +244,51 @@
2131         response = self.client.post('/test_client_regress/file_upload/', post_data)
2132         self.assertEqual(response.status_code, 200)
2133 
2134+    def test_large_upload(self):
2135+        import tempfile
2136+        dir = tempfile.gettempdir()
2137+
2138+        (fd, name1) = tempfile.mkstemp(suffix='.file1', dir=dir)
2139+        file1 = os.fdopen(fd, 'w+b')
2140+        file1.write('a' * (2 ** 21))
2141+        file1.seek(0)
2142+
2143+        (fd, name2) = tempfile.mkstemp(suffix='.file2', dir=dir)
2144+        file2 = os.fdopen(fd, 'w+b')
2145+        file2.write('a' * (10 * 2 ** 20))
2146+        file2.seek(0)
2147+
2148+        # This file contains chinese symbols for a name.
2149+        name3 = os.path.join(dir, u'test_&#20013;&#25991;_Orl\u00e9ans.jpg')
2150+        file3 = open(name3, 'w+b')
2151+        file3.write('b' * (2 ** 10))
2152+        file3.seek(0)
2153+
2154+        post_data = {
2155+            'name': 'Ringo',
2156+            'file_field1': file1,
2157+            'file_field2': file2,
2158+            'file_unicode': file3,
2159+            }
2160+
2161+        for key in post_data.keys():
2162+            try:
2163+                post_data[key + '_hash'] = sha.new(post_data[key].read()).hexdigest()
2164+                post_data[key].seek(0)
2165+            except AttributeError:
2166+                post_data[key + '_hash'] = sha.new(post_data[key]).hexdigest()
2167+
2168+        response = self.client.post('/test_client_regress/file_upload_verify/', post_data)
2169+
2170+        for name in (name1, name2, name3):
2171+            try:
2172+                os.unlink(name)
2173+            except:
2174+                pass
2175+
2176+        self.assertEqual(response.status_code, 200)
2177+
2178+
2179 class LoginTests(TestCase):
2180     fixtures = ['testdata']
2181 
2182Index: tests/regressiontests/test_client_regress/urls.py
2183===================================================================
2184--- tests/regressiontests/test_client_regress/urls.py   (revision 7363)
2185+++ tests/regressiontests/test_client_regress/urls.py   (working copy)
2186@@ -4,6 +4,7 @@
2187 urlpatterns = patterns('',
2188     (r'^no_template_view/$', views.no_template_view),
2189     (r'^file_upload/$', views.file_upload_view),
2190+    (r'^file_upload_verify/$', views.file_upload_view_verify),
2191     (r'^get_view/$', views.get_view),
2192     url(r'^arg_view/(?P<name>.+)/$', views.view_with_argument, name='arg_view'),
2193     (r'^login_protected_redirect_view/$', views.login_protected_redirect_view)
2194Index: tests/regressiontests/datastructures/tests.py
2195===================================================================
2196--- tests/regressiontests/datastructures/tests.py       (revision 7363)
2197+++ tests/regressiontests/datastructures/tests.py       (working copy)
2198@@ -117,12 +117,23 @@
2199 >>> d['person']['2']['firstname']
2200 ['Adrian']
2201 
2202-### FileDict ################################################################
2203-
2204->>> d = FileDict({'content': 'once upon a time...'})
2205+### ImmutableList ################################################################
2206+>>> d = ImmutableList(range(10))
2207+>>> d.sort()
2208+Traceback (most recent call last):
2209+  File "<stdin>", line 1, in <module>
2210+  File "/var/lib/python-support/python2.5/django/utils/datastructures.py", line 359, in complain
2211+    raise AttributeError, self.warning
2212+AttributeError: ImmutableList object is immutable.
2213 >>> repr(d)
2214-"{'content': '<omitted>'}"
2215->>> d = FileDict({'other-key': 'once upon a time...'})
2216->>> repr(d)
2217-"{'other-key': 'once upon a time...'}"
2218+'(0, 1, 2, 3, 4, 5, 6, 7, 8, 9)'
2219+>>> d = ImmutableList(range(10), warning="Object is immutable!")
2220+>>> d[1]
2221+1
2222+>>> d[1] = 'test'
2223+Traceback (most recent call last):
2224+  File "<stdin>", line 1, in <module>
2225+  File "/var/lib/python-support/python2.5/django/utils/datastructures.py", line 359, in complain
2226+    raise AttributeError, self.warning
2227+AttributeError: Object is immutable!
2228 """
2229Index: AUTHORS
2230===================================================================
2231--- AUTHORS     (revision 7363)
2232+++ AUTHORS     (working copy)
2233@@ -58,7 +58,7 @@
2234     Jökull Sólberg Auðunsson <jokullsolberg@gmail.com>
2235     Arthur <avandorp@gmail.com>
2236     David Avsajanishvili <avsd05@gmail.com>
2237-    axiak@mit.edu
2238+    Mike Axiak <axiak@mit.edu>
2239     Niran Babalola <niran@niran.org>
2240     Morten Bagai <m@bagai.com>
2241     Mikaël Barbero <mikael.barbero nospam at nospam free.fr>
2242@@ -135,6 +135,7 @@
2243     Marc Fargas <telenieko@telenieko.com>
2244     Szilveszter Farkas <szilveszter.farkas@gmail.com>
2245     favo@exoweb.net
2246+    fdr <drfarina@gmail.com>
2247     Dmitri Fedortchenko <zeraien@gmail.com>
2248     Bill Fenner <fenner@gmail.com>
2249     Stefane Fermgier <sf@fermigier.com>
2250Index: docs/upload_handling.txt
2251===================================================================
2252--- docs/upload_handling.txt    (revision 0)
2253+++ docs/upload_handling.txt    (revision 0)
2254@@ -0,0 +1,289 @@
2255+============
2256+File Uploads
2257+============
2258+
2259+**New in Django development version**
2260+
2261+Most Web sites wouldn't be complete without a way to upload files. Before the
2262+file even gets into ``request.FILES`` (discussed in the `Request
2263+and response objects`_ documentation) Django has to decide where to put the
2264+all the incoming data. This document describes how Django deals with this
2265+stage of the upload process.
2266+
2267+
2268+.. _Request and response objects: ../request_response/#attributes
2269+
2270+Default Behavior
2271+================
2272+
2273+If you set up a quick model that contains a `FileField`_ and use the `Admin
2274+interface`_ you can quickly create a form to upload files to your site.
2275+By default, if the upload is smaller than *2.5 Megabytes* in size, Django
2276+will hold the entire contents of the upload in memory, and the file will
2277+quickly be saved to its final destination (as defined by
2278+`settings.MEDIA_ROOT`_) without any intermediary.
2279+
2280+If the entire upload is larger than 2.5 Megabytes, then it will -- by
2281+default -- write the contents of the uploaded file to a temporary file
2282+in your operating system's default temporary directory. On a posix platform,
2283+this means that you could expect a file similar to ``/tmp/tmpzfp6I6.upload``
2284+to be generated and filled during the upload process.
2285+
2286+.. note::
2287+    You may find that the default temporary directory is not writable.
2288+    This is especially true if you are on shared hosting. If this is
2289+    the case, Django will raise an error when you try to upload. Please
2290+    read below in `Extending the Default`_ to change this directory.
2291+
2292+
2293+.. _FileField: ../model-api/#filefield
2294+.. _Admin interface: ../tutorial02/#activate-the-admin-site
2295+.. _settings.MEDIA_ROOT: ../settings/#media-root
2296+
2297+Extending the Default
2298+=====================
2299+
2300+If you have quite a bit of memory, you may decide that you want Django to
2301+stream only if the entire upload exceeds 10 Megabytes. You may even
2302+decide that whatever directory is the platform default is not suitable
2303+for your project. If this is the case, Django provides these two settings:
2304+
2305+    =============================== ======================================
2306+    Setting                         Description
2307+    =============================== ======================================
2308+    ``FILE_UPLOAD_MAX_MEMORY_SIZE`` The maximum size of a request in bytes
2309+                                    for which Django will try to load the
2310+                                    entire upload contents in memory.
2311+
2312+    ``FILE_UPLOAD_TEMP_DIR``        The directory on the file system where
2313+                                    uploaded contents will be temporarily
2314+                                    stored if not completely in memory.
2315+                                    E.g.: ``"/tmp"``
2316+    =============================== ======================================
2317+
2318+There is one final setting -- ``FILE_UPLOAD_HANDLERS`` -- which allows complete
2319+customization of the upload process.
2320+
2321+Upload Handlers
2322+===============
2323+
2324+Through upload handlers Django provides the flexibility to extend the
2325+upload process beyond simple storage. You can use custom handlers to enforce
2326+user-level quotas, compress data on the fly, render progress bars, and
2327+even send data to another warehouse directly without storing it locally!
2328+
2329+There are two pieces to the Django upload handling: the upload handler and the
2330+uploaded file. These are both represented by python classes --
2331+``FileUploadHandler`` and ``UploadedFile`` respectively. Throughout the lifetime
2332+of the upload process, Django will call on the upload handler to handle the
2333+upload, while the upload handler is expected to return an ``UploadedFile``
2334+object at the end of a file's upload.
2335+
2336+Setting Default Upload Handlers for your Project
2337+------------------------------------------------
2338+
2339+Similar to `Middleware`_, upload handlers have an order that is initially
2340+defined in ``settings.FILE_UPLOAD_HANDLERS``. The default value is::
2341+
2342+    ("django.core.files.fileuploadhandler.MemoryFileUploadHandler",
2343+     "django.core.files.fileuploadhandler.TemporaryFileUploadHandler",)
2344+
2345+This literally means: Try putting the upload in memory first and failing
2346+that put the upload in a temporary file. However, this behavior is
2347+completely dependent on how each of those handlers is defined. For instance,
2348+just because there isn't an example, there's no reason an upload handler
2349+could run first recording information.
2350+
2351+.. _Middleware: ../middleware/
2352+
2353+Modifying your Upload Handlers Dynamically
2354+------------------------------------------
2355+
2356+During the lifetime of your project, you may realize that a particular
2357+view or views require different uploading behavior. For this reason,
2358+the ``request`` object contains a list of handlers
2359+(``request.upload_handlers``) that will be called in order. To append
2360+a handler to the list, you would append to it like any other list.
2361+For example, suppose you had an ``ProgressBarUploadHandler`` class.
2362+To append it to your upload handlers you would write::
2363+
2364+    request.upload_handlers.append(ProgressBarUploadHandler())
2365+
2366+However, since the progress bar handler would probably need to run before the
2367+other handlers get a chance, you'd probably want to insert it. That is, you'd
2368+write::
2369+
2370+   request.upload_handlers.insert(0, ProgressBarUploadHandler())
2371+
2372+If you want to replace the upload handlers completely, you can just assign a new
2373+list::
2374+
2375+   request.upload_handlers = [ProgressBarUploadHandler()]
2376+
2377+And all Django will do is keep a progress, but do nothing with the file itself!
2378+One more note: After the upload has completed, you are no longer allowed to
2379+assign or modify this list, and Django will raise an error if you try to modify
2380+this list after an upload.
2381+
2382+Writing a File Upload Handler
2383+-----------------------------
2384+
2385+All file upload handlers are subclasses of ``FileUploadHandler``, found in
2386+``django.core.files.fileuploadhandler``. To create your handler, you need to
2387+define the required methods followed by the methods that make most sense to you:
2388+
2389+chunk_size
2390+~~~~~~~~~~
2391+
2392+This is an integer attribute that specifies how large the chunks we should put
2393+into memory from the network are. The chunk sizes should be divisible by ``4``
2394+and should not exceed ``2 ** 31 - 1`` in size. When there are multiple chunk
2395+sizes provided by multiple handlers, Django will use the smallest chunk size.
2396+
2397+new_file
2398+~~~~~~~~
2399+
2400+This signals that a new file is starting. You can initialize a file or whatever
2401+else is needed on a new file upload.
2402+
2403+Interface: ``new_file(self, field_name, file_name, content_type, content_length,
2404+charset)``
2405+
2406+``field_name`` is a string name of the field this was POSTed as.
2407+
2408+``file_name`` is the unicode filename that was provided by the browser.
2409+
2410+``content_type`` is the MIME type provided by the browser -- E.g.
2411+``'image/jpeg'``.
2412+
2413+``content_length`` is the length of the image given by the browser if provided,
2414+``None`` otherwise.
2415+
2416+``charset`` is the charset given by the browser if provided, ``None`` otherwise.
2417+
2418+Returns: ``None`` if you want other handlers to get ``new_file`` called.
2419+Something nonzero if you don't want subsequent handlers to get a chance.
2420+
2421+receive_data_chunk
2422+~~~~~~~~~~~~~~~~~~
2423+*required*
2424+
2425+This method is used to do something with the new chunk of data. For example: The
2426+``TemporaryFileUploadHandler`` takes the data and writes it to disk.
2427+
2428+Interface: ``receive_data_chunk(self, raw_data, start, stop)``
2429+
2430+``raw_data`` is a byte string containing the uploaded data.
2431+
2432+``start`` is the byte number where the chunk starts.
2433+
2434+``stop`` is the byte number where the chunk stops.
2435+
2436+Returns: ``None`` if you don't want the subsequent upload handlers to receive
2437+the data. Whatever else you return gets fed into the subsequent upload handlers'
2438+``receive_data_chunk`` method.
2439+
2440+Exceptions: If you raise a ``StopUpload`` or a ``SkipFile`` exception, the
2441+upload will abort or the file will be skipped respectively.
2442+
2443+file_complete
2444+~~~~~~~~~~~~~
2445+*required*
2446+
2447+This method defines when a function has finished uploading and gets packaged
2448+into an ``UploadedFile`` object.
2449+
2450+Interface: ``file_complete(self, file_size)``
2451+
2452+``file_size`` is the number of bytes you have received for this file.
2453+
2454+Returns: An ``UploadedFile`` object to set in the ``request.FILES`` dictionary.
2455+``None`` if you want the subsequent upload handlers to get an ``UploadedFile``
2456+object.
2457+
2458+upload_complete
2459+~~~~~~~~~~~~~~~
2460+
2461+This method defines when the entire upload has completed.
2462+
2463+Interface: ``upload_complete(self)``
2464+
2465+handle_raw_input
2466+~~~~~~~~~~~~~~~~
2467+
2468+This method allows the handler to completely override the parsing of the raw
2469+HTTP-layered input.
2470+
2471+Interface: ``handle_raw_input(self, input_data, META, content_length, boundary,
2472+encoding)``
2473+
2474+``input_data`` is a file-like object that supports the read operation.
2475+
2476+``META`` is the same object as ``request.META``.
2477+
2478+``content_length`` is the length of the data in ``input_data``. Don't read more
2479+than ``content_length`` bytes from ``input_data``.
2480+
2481+``boundary`` is the MIME boundary for this request.
2482+
2483+``encoding`` is the encoding of the request.
2484+
2485+Returns: ``None`` if you want to go on to the next stage, ``(POST, FILES)`` if
2486+you want to return the new data structures suitable for the request directly.
2487+
2488+Defining an Uploaded File
2489+-------------------------
2490+
2491+All file upload handlers are subclasses of ``UploadedFile``, found in
2492+``django.core.files.uploadedfile``. The uploaded file object is returned by the
2493+handler above in ``file_complete``. To create your own uploaded file class, you
2494+need to define the required methods followed by the methods that make most sense
2495+to you:
2496+
2497+read
2498+~~~~
2499+*required*
2500+
2501+Interface: ``read(self, num_bytes=None)``
2502+
2503+Returns: A byte string of length ``num_bytes`` (or the size of the file if
2504+``num_bytes`` is not supplied).
2505+
2506+chunk
2507+~~~~~
2508+
2509+A generator to yield small chunks from the file. With the ``read()`` defined,
2510+the ``UploadedFile`` class already defines a ``chunk()`` method that's probably
2511+suitable.
2512+
2513+Interface: ``chunk(self, chunk_size=None)``
2514+
2515+multiple_chunks
2516+~~~~~~~~~~~~~~~
2517+
2518+Interface: ``multiple_chunks(self, chunk_size=None)``
2519+
2520+Returns: ``True`` or ``False`` depending on whether or not the user of this file
2521+can expect more than one chunk when calling ``chunk(self, chunk_size)``. If all
2522+the data is in memory, you should return ``False``.
2523+
2524+temporary_file_path
2525+~~~~~~~~~~~~~~~~~~~
2526+
2527+If defined, this method should return the file path of the file on the operating
2528+system. This will let users move files rather than write to new files if the
2529+option is available.
2530+
2531+Interface: ``temporary_file_path(self)``
2532+
2533+Returns: A file path in a file system on the local computer.
2534+
2535+upload_errors
2536+~~~~~~~~~~~~~
2537+
2538+If defined, this method should return a string describing errors that occured
2539+during uploads.
2540+
2541+Interface: ``upload_errors(self)``
2542+
2543+Returns: A unicode string describing an error that occured with the file's upload.
2544Index: docs/settings.txt
2545===================================================================
2546--- docs/settings.txt   (revision 7363)
2547+++ docs/settings.txt   (working copy)
2548@@ -513,6 +513,36 @@
2549 The character encoding used to decode any files read from disk. This includes
2550 template files and initial SQL data files.
2551 
2552+FILE_UPLOAD_HANDLERS
2553+--------------------
2554+
2555+**New in Django development version**
2556+
2557+Default::
2558+
2559+    ("django.core.files.fileuploadhandler.MemoryFileUploadHandler",
2560+     "django.core.files.fileuploadhandler.TemporaryFileUploadHandler",)
2561+
2562+A tuple of handlers to use for uploading.
2563+
2564+FILE_UPLOAD_MAX_MEMORY_SIZE
2565+---------------------------
2566+
2567+**New in Django development version**
2568+
2569+Default: ``2621440``
2570+
2571+The maximum size (in bytes) that an upload will be before it gets streamed to the file system.
2572+
2573+FILE_UPLOAD_TEMP_DIR
2574+--------------------
2575+
2576+**New in Django development version**
2577+
2578+Default: ``None``
2579+
2580+The directory to store data temporarily while uploading files. If ``None``, Django will use the standard temporary directory for the operating system. For example, this will default to '/tmp' on *nix-style operating systems.
2581+
2582 FIXTURE_DIRS
2583 -------------
2584 
2585Index: docs/newforms.txt
2586===================================================================
2587--- docs/newforms.txt   (revision 7363)
2588+++ docs/newforms.txt   (working copy)
2589@@ -805,12 +805,12 @@
2590 need to bind the file data containing the mugshot image::
2591 
2592     # Bound form with an image field
2593+    >>> from django.core.files.uploadedfile import SimpleUploadedFile
2594     >>> data = {'subject': 'hello',
2595     ...         'message': 'Hi there',
2596     ...         'sender': 'foo@example.com',
2597     ...         'cc_myself': True}
2598-    >>> file_data = {'mugshot': {'filename':'face.jpg'
2599-    ...                          'content': <file data>}}
2600+    >>> file_data = {'mugshot': SimpleUploadedFile('face.jpg', <file data>)}
2601     >>> f = ContactFormWithMugshot(data, file_data)
2602 
2603 In practice, you will usually specify ``request.FILES`` as the source