Code

Ticket #2070: 2070_revision7351_latest.diff

File 2070_revision7351_latest.diff, 67.7 KB (added by axiak, 6 years ago)

Latest patch for 2070...includes everything (see comment)

Line 
1Index: django/http/multipartparser.py
2===================================================================
3--- django/http/multipartparser.py      (revision 0)
4+++ django/http/multipartparser.py      (revision 0)
5@@ -0,0 +1,585 @@
6+"""
7+MultiPart parsing for file uploads.
8+
9+This object will take the file upload headers
10+and the file upload handler and chunk the upload
11+data for the handler to deal with.
12+"""
13+from django.utils.datastructures import MultiValueDict
14+from django.utils.encoding import force_unicode
15+
16+__all__ = ('MultiPartParser','MultiPartParserError','InputStreamExhausted')
17+
18+class MultiPartParserError(Exception):
19+    pass
20+
21+class InputStreamExhausted(Exception):
22+    """ No more reads are allowed from this device. """
23+    pass
24+
25+class MultiPartParser(object):
26+    """
27+    A rfc2388 multipart/form-data parser.
28+
29+    parse() reads the input stream in chunk_size chunks and returns a
30+    tuple of (POST MultiValueDict, FILES MultiValueDict). If
31+    file_upload_dir is defined files will be streamed to temporary
32+    files in the specified directory.
33+    """
34+    def __init__(self, META, input_data, upload_handlers, encoding=None):
35+        """
36+        Initialize the MultiPartParser object.
37+
38+        *META* -- The standard META dictionary in Django request objects.
39+        *input_data* -- The raw post data, as a bytestring.
40+        *upload_handler* -- An object of type UploadHandler
41+                            that performs operations on the uploaded
42+                            data.
43+        *encoding* -- The encoding with which to treat the incoming data.
44+        """
45+        # Import cgi utilities for (near) future use.
46+        global parse_header, valid_boundary, settings
47+        from django.conf import settings
48+        from cgi import valid_boundary, parse_header
49+
50+        #######
51+        # Check basic headers
52+        #######
53+
54+        #
55+        # Content-Type should containt multipart and the boundary information.
56+        ####
57+
58+        content_type = META.get('HTTP_CONTENT_TYPE', META.get('CONTENT_TYPE', ''))
59+        if not content_type.startswith('multipart/'):
60+            raise MultiPartParserError('Invalid Content-Type: %s' %
61+                                       content_type)
62+
63+        # Parse the header to get the boundary to split the parts.
64+        ctypes, opts = parse_header(content_type)
65+        boundary = opts.get('boundary')
66+        if not boundary or not valid_boundary(boundary):
67+            raise MultiPartParserError('Invalid boundary in multipart: %s' %
68+                                       boundary)
69+
70+
71+        #
72+        # Content-Length should contain the length of the body we are about
73+        # to receive.
74+        ####
75+        try:
76+            content_length = int(META.get('HTTP_CONTENT_LENGTH',
77+                                          META.get('CONTENT_LENGTH',0)))
78+        except (ValueError, TypeError):
79+            # For now set it to 0...we'll try again later on down.
80+            content_length = 0
81+
82+        # If we have better knowledge of how much
83+        # data is remaining in the request stream,
84+        # we should use that. (modpython for instance)
85+        #try:
86+        #    remaining = input_data.remaining
87+        #    if remaining is not None and \
88+        #            (content_length is None or remaining < content_length):
89+        #        content_length = remaining
90+        #except AttributeError:
91+        #    pass
92+
93+        if not content_length:
94+            # This means we shouldn't continue...raise an error.
95+            raise MultiPartParserError("Invalid content length: %r" % content_length)
96+
97+        self._boundary = boundary
98+        self._input_data = input_data
99+
100+        # For compatibility with low-level network APIs (with 32-bit integers),
101+        # the chunk size should be < 2^31:
102+        self._chunk_size = min(2147483647, *[x.chunk_size for x in upload_handlers
103+                                            if x.chunk_size])
104+
105+        self._meta = META
106+        self._encoding = encoding or settings.DEFAULT_CHARSET
107+        self._content_length = content_length
108+        self._upload_handlers = upload_handlers
109+
110+    def parse(self):
111+        """
112+        Parse the POST data and break it into a FILES MultiValueDict
113+        and a POST MultiValueDict.
114+
115+           *returns* -- A tuple containing the POST and FILES dictionary,
116+                        respectively.
117+        """
118+        from base64 import b64decode
119+        from django.core.files.fileuploadhandler import StopUpload, SkipFile
120+        from django.http import QueryDict
121+
122+        encoding = self._encoding
123+        handlers = self._upload_handlers
124+
125+        limited_input_data = LimitBytes(self._input_data, self._content_length)
126+
127+        # See if the handler will want to take care of the parsing.
128+        # This allows overriding everything if somebody wants it.
129+        for handler in handlers:
130+            result = handler.handle_raw_input(limited_input_data,
131+                                              self._meta,
132+                                              self._content_length,
133+                                              self._boundary,
134+                                              encoding)
135+            if result is not None:
136+                return result[0], result[1]
137+
138+        # Create the data structures to be used later.
139+        self._post = QueryDict('', mutable=True)
140+        self._files = MultiValueDict()
141+
142+        # Instantiate the parser and stream:
143+        stream = LazyStream(ChunkIter(limited_input_data, self._chunk_size))
144+        for item_type, meta_data, stream in Parser(stream, self._boundary):
145+            try:
146+                disposition = meta_data['content-disposition'][1]
147+                field_name = disposition['name'].strip()
148+            except (KeyError, IndexError, AttributeError):
149+                continue
150+
151+            transfer_encoding = meta_data.get('content-transfer-encoding')
152+
153+            field_name = force_unicode(field_name, encoding, errors='replace')
154+
155+            if item_type == 'FIELD':
156+                # This is a post field, we can just set it in the post
157+                if transfer_encoding == 'base64':
158+                    raw_data = stream.read()
159+                    try:
160+                        data = b64decode(raw_data)
161+                    except TypeError:
162+                        data = raw_data
163+                else:
164+                    data = stream.read()
165+
166+                self._post.appendlist(field_name,
167+                                      force_unicode(data, encoding, errors='replace'))
168+            elif item_type == 'FILE':
169+                # This is a file, use the handler...
170+                file_successful = True
171+                file_name = self.IE_sanitize(disposition.get('filename'))
172+                if not file_name:
173+                    continue
174+
175+                file_name = force_unicode(file_name, encoding, errors='replace')
176+
177+                content_type = meta_data.get('content-type', ('',))[0].strip()
178+                try:
179+                    charset = meta_data.get('content-type', (0,{}))[1].get('charset', None)
180+                except:
181+                    charset = None
182+
183+                try:
184+                    content_length = int(meta_data.get('content-length')[0])
185+                except (IndexError, TypeError, ValueError):
186+                    content_length = None
187+
188+                counter = 0
189+                try:
190+                    for handler in handlers:
191+                        retval = handler.new_file(field_name, file_name,
192+                                                  content_type, content_length,
193+                                                  charset)
194+                        if retval:
195+                            break
196+
197+                    for chunk in stream:
198+                        if transfer_encoding == 'base64':
199+                            # We only special-case base64 transfer encoding
200+                            try:
201+                                chunk = b64decode(chunk)
202+                            except TypeError, e:
203+                                raise MultiValueParseError("Could not decode base64 data: %r" % e)
204+
205+                        chunk_length = len(chunk)
206+                        counter += chunk_length
207+                        for handler in handlers:
208+                            retval = handler.receive_data_chunk(chunk,
209+                                                                counter - chunk_length,
210+                                                                counter)
211+                            if retval:
212+                                break
213+
214+                except (StopUpload, SkipFile), e:
215+                    file_successful = False
216+                    if isinstance(e, SkipFile):
217+                        # Just use up the rest of this file...
218+                        stream.exhaust()
219+                    elif isinstance(e, StopUpload):
220+                        # Abort the parsing and break
221+                        parser.abort()
222+                        break
223+                else:
224+                    # Only do this if the handler didn't raise an abort error
225+                    for handler in handlers:
226+                        file_obj = handler.file_complete(counter)
227+                        if file_obj:
228+                            # If it returns a file object, then set the files dict.
229+                            self._files.appendlist(force_unicode(field_name,
230+                                                                 encoding,
231+                                                                 errors='replace'),
232+                                                   file_obj)
233+                            break
234+            else:
235+                # If this is neither a FIELD or a FILE, just exhaust the stream.
236+                stream.exhuast()
237+
238+        # Make sure that the request data is all fed
239+        limited_input_data.exhaust()
240+
241+        # Signal that the upload has completed.
242+        for handler in handlers:
243+            retval = handler.upload_complete()
244+            if retval:
245+                break
246+
247+        return self._post, self._files
248+
249+    def IE_sanitize(self, filename):
250+        """cleanup filename from IE full paths"""
251+        return filename and filename[filename.rfind("\\")+1:].strip()
252+
253+
254+class LazyStream(object):
255+    def __init__(self, producer, length=None):
256+        """
257+        Every LazyStream must have a producer when instantiated.
258+
259+        A producer is an iterable that returns a string each time it
260+        is called.
261+        """
262+        self._producer = producer
263+        self._empty = False
264+        self._leftover = ''
265+        self.length = length
266+        self.position = 0
267+        self._remaining = length
268+
269+    def tell(self):
270+        return self.position
271+
272+    def read(self, size=None):
273+        def parts():
274+            remaining = (size is not None and [size] or [self._remaining])[0]
275+            # do the whole thing in one shot if no limit was provided.
276+            if remaining is None:
277+                yield ''.join(self)
278+                return
279+
280+            # otherwise do some bookkeeping to return exactly enough
281+            # of the stream and stashing any extra content we get from
282+            # the producer
283+            while remaining != 0:
284+                assert remaining > 0, 'remaining bytes to read should never go negative'
285+
286+                chunk = self.next()
287+
288+                emitting = chunk[:remaining]
289+                self.unget(chunk[remaining:])
290+                remaining -= len(emitting)
291+                yield emitting
292+
293+        out = ''.join(parts())
294+        self.position += len(out)
295+        return out
296+
297+    def next(self):
298+        """
299+        Used when the exact number of bytes to read is unimportant.
300+
301+        This procedure just returns whatever is chunk is conveniently
302+        returned from the iterator instead. Useful to avoid
303+        unnecessary bookkeeping if performance is an issue.
304+        """
305+        if self._leftover:
306+            output = self._leftover
307+            self.position += len(output)
308+            self._leftover = ''
309+            return output
310+        else:
311+            output = self._producer.next()
312+            self.position += len(output)
313+            return output
314+
315+    def close(self):
316+        """
317+        Used to invalidate/disable this lazy stream.
318+
319+        Replaces the producer with an empty list. Any leftover bytes
320+        that have already been read will still be reported upon read()
321+        and/or next().
322+        """
323+        self._producer = []
324+
325+    def __iter__(self):
326+        return self
327+
328+    def unget(self, bytes):
329+        """
330+        Places bytes back onto the front of the lazy stream.
331+
332+        Future calls to read() will return those bytes first. The
333+        stream position and thus tell() will be rewound.
334+        """
335+        self.position -= len(bytes)
336+        self._leftover = ''.join([bytes, self._leftover])
337+
338+    def exhaust(self):
339+        """
340+        Exhausts the entire underlying stream.
341+
342+        Useful for skipping and advancing sections.
343+        """
344+        for thing in self:
345+            pass
346+
347+
348+class ChunkIter(object):
349+    def __init__(self, flo, chunk_size=1024**2):
350+        self.flo = flo
351+        self.chunk_size = chunk_size
352+
353+    def next(self):
354+        try:
355+            data = self.flo.read(self.chunk_size)
356+        except InputStreamExhausted:
357+            raise StopIteration
358+        if data:
359+            return data
360+        else:
361+            raise StopIteration
362+
363+    def __iter__(self):
364+        return self
365+
366+
367+class LimitBytes(object):
368+    """ Limit bytes for a file object. """
369+    def __init__(self, fileobject, length):
370+        self._file = fileobject
371+        self.remaining = length
372+
373+    def read(self, num_bytes=None):
374+        """
375+        Read data from the underlying file.
376+        If you ask for too much or there isn't anything left,
377+        this will raise an InputStreamExhausted error.
378+        """
379+        if self.remaining <= 0:
380+            raise InputStreamExhausted()
381+        if num_bytes is None:
382+            num_bytes = self.remaining
383+        else:
384+            num_bytes = min(num_bytes, self.remaining)
385+        self.remaining -= num_bytes
386+        return self._file.read(num_bytes)
387+
388+    def exhaust(self):
389+        """
390+        Exhaust this file until all of the bytes it was limited by
391+        have been read.
392+        """
393+        while self.remaining > 0:
394+            num_bytes = min(self.remaining, 16384)
395+            __ = self._file.read(num_bytes)
396+            self.remaining -= num_bytes
397+
398+
399+class InterBoundaryIter(object):
400+    """
401+    A Producer that will iterate over boundaries.
402+    """
403+    def __init__(self, stream, boundary):
404+        self._stream = stream
405+        self._boundary = boundary
406+
407+    def __iter__(self):
408+        return self
409+
410+    def next(self):
411+        try:
412+            return LazyStream(BoundaryIter(self._stream, self._boundary))
413+        except InputStreamExhausted:
414+            raise StopIteration
415+
416+class BoundaryIter(object):
417+    """
418+    A Producer that is sensitive to boundaries.
419+
420+    Will happily yield bytes until a boundary is found. Will yield the
421+    bytes before the boundary, throw away the boundary bytes
422+    themselves, and push the post-boundary bytes back on the stream.
423+
424+    The future calls to .next() after locating the boundary will raise
425+    a StopIteration exception.
426+    """
427+    def __init__(self, stream, boundary):
428+        self._stream = stream
429+        self._boundary = boundary
430+        self._done = False
431+        # rollback an additional six bytes because the format is like
432+        # this: CRLF<boundary>[--CRLF]
433+        self._rollback = len(boundary) + 6
434+
435+        # Try to use mx fast string search if available. Otherwise
436+        # use Python find. Wrap the latter for consistency.
437+        unused_char = self._stream.read(1)
438+        if not unused_char:
439+            raise InputStreamExhausted
440+        self._stream.unget(unused_char)
441+        try:
442+            from mx.TextTools import FS
443+            self._fs = FS(boundary).find
444+        except ImportError:
445+            self._fs = lambda data: data.find(boundary)
446+
447+    def __iter__(self):
448+        return self
449+
450+    def next(self):
451+        if self._done:
452+            raise StopIteration
453+
454+        stream = self._stream
455+        rollback = self._rollback
456+
457+        bytes_read = 0
458+        chunks = []
459+        for bytes in stream:
460+            bytes_read += len(bytes)
461+            chunks.append(bytes)
462+            if bytes_read > rollback:
463+                break
464+            if not bytes:
465+                break
466+        else:
467+            self._done = True
468+
469+        if not chunks:
470+            raise StopIteration
471+
472+        chunk = ''.join(chunks)
473+
474+        boundary = self._find_boundary(chunk, len(chunk) < self._rollback)
475+
476+
477+        if boundary:
478+            end, next = boundary
479+            stream.unget(chunk[next:])
480+            self._done = True
481+            return chunk[:end]
482+        else:
483+            # make sure we dont treat a partial boundary (and
484+            # its separators) as data
485+            if not chunk[:-rollback]:# and len(chunk) >= (len(self._boundary) + 6):
486+                # There's nothing left, we should just return and mark as done.
487+                self._done = True
488+                return chunk
489+            else:
490+                stream.unget(chunk[-rollback:])
491+                return chunk[:-rollback]
492+
493+    def _find_boundary(self, data, eof = False):
494+        """
495+        Finds a multipart boundary in data.
496+
497+        Should no boundry exist in the data None is returned
498+        instead. Otherwise a tuple containing
499+        the indices of the following are returned:
500+
501+         * the end of current encapsulation
502+
503+         * the start of the next encapsulation
504+        """
505+        index = self._fs(data)
506+        if index < 0:
507+            return None
508+        else:
509+            end = index
510+            next = index + len(self._boundary)
511+            data_len = len(data) - 1
512+            # backup over CRLF
513+            if data[max(0,end-1)] == '\n': end -= 1
514+            if data[max(0,end-1)] == '\r': end -= 1
515+            # skip over --CRLF
516+            if data[min(data_len,next)] == '-': next += 1
517+            if data[min(data_len,next)] == '-': next += 1
518+            if data[min(data_len,next)] == '\r': next += 1
519+            if data[min(data_len,next)] == '\n': next += 1
520+            return end, next
521+
522+def ParseBoundaryStream(stream, max_header_size):
523+        """
524+        Parses one and exactly one stream that encapsulates a boundary.
525+        """
526+        # Stream at beginning of header, look for end of header
527+        # and parse it if found. The header must fit within one
528+        # chunk.
529+        chunk = stream.read(max_header_size)
530+        # 'find' returns the top of these four bytes, so we'll
531+        # need to munch them later to prevent them from polluting
532+        # the payload.
533+        header_end = chunk.find('\r\n\r\n')
534+
535+        def parse_header(line):
536+            from cgi import parse_header
537+            main_value_pair, params = parse_header(line)
538+            try:
539+                name, value = main_value_pair.split(':', 1)
540+            except:
541+                raise ValueError("Invalid header: %r" % line)
542+            return name, (value, params)
543+
544+        if header_end == -1:
545+            # we find no header, so we just mark this fact and pass on
546+            # the stream verbatim
547+            stream.unget(chunk)
548+            return ('RAW', {}, stream)
549+
550+        header = chunk[:header_end]
551+
552+        # here we place any excess chunk back onto the stream, as
553+        # well as throwing away the CRLFCRLF bytes from above.
554+        stream.unget(chunk[header_end + 4:])
555+
556+        is_file_field = False
557+        outdict = {}
558+
559+        # eliminate blank lines
560+        for line in header.split('\r\n'):
561+            # This terminology ("main value" and "dictionary of
562+            # parameters") is from the Python docs.
563+            name, (value, params) = parse_header(line)
564+            if name == 'content-disposition' and params.get('filename'):
565+                is_file_field = True
566+
567+            outdict[name] = value, params
568+
569+        if is_file_field:
570+            return ('FILE', outdict, stream)
571+        else:
572+            return ('FIELD', outdict, stream)
573+
574+
575+class Parser(object):
576+    def __init__(self, stream, boundary):
577+        self._stream = stream
578+        self._separator = '--' + boundary
579+
580+    def __iter__(self):
581+
582+        boundarystream = InterBoundaryIter(self._stream,
583+                                           self._separator)
584+
585+        for sub_stream in boundarystream:
586+            # Iterate over each part
587+            yield ParseBoundaryStream(sub_stream, 1024)
588+
589+
590+
591Index: django/http/__init__.py
592===================================================================
593--- django/http/__init__.py     (revision 7351)
594+++ django/http/__init__.py     (working copy)
595@@ -11,7 +11,7 @@
596 
597 from django.utils.datastructures import MultiValueDict, FileDict
598 from django.utils.encoding import smart_str, iri_to_uri, force_unicode
599-
600+from django.http.multipartparser import MultiPartParser
601 from utils import *
602 
603 RESERVED_CHARS="!*'();:@&=+$,/?%#[]"
604@@ -30,6 +30,7 @@
605         self.GET, self.POST, self.COOKIES, self.META, self.FILES = {}, {}, {}, {}, {}
606         self.path = ''
607         self.method = None
608+        self._upload_handlers = []
609 
610     def __repr__(self):
611         return '<HttpRequest\nGET:%s,\nPOST:%s,\nCOOKIES:%s,\nMETA:%s>' % \
612@@ -102,40 +103,35 @@
613 
614     encoding = property(_get_encoding, _set_encoding)
615 
616-def parse_file_upload(header_dict, post_data):
617-    """Returns a tuple of (POST QueryDict, FILES MultiValueDict)."""
618-    import email, email.Message
619-    from cgi import parse_header
620-    raw_message = '\r\n'.join(['%s:%s' % pair for pair in header_dict.items()])
621-    raw_message += '\r\n\r\n' + post_data
622-    msg = email.message_from_string(raw_message)
623-    POST = QueryDict('', mutable=True)
624-    FILES = MultiValueDict()
625-    for submessage in msg.get_payload():
626-        if submessage and isinstance(submessage, email.Message.Message):
627-            name_dict = parse_header(submessage['Content-Disposition'])[1]
628-            # name_dict is something like {'name': 'file', 'filename': 'test.txt'} for file uploads
629-            # or {'name': 'blah'} for POST fields
630-            # We assume all uploaded files have a 'filename' set.
631-            if 'filename' in name_dict:
632-                assert type([]) != type(submessage.get_payload()), "Nested MIME messages are not supported"
633-                if not name_dict['filename'].strip():
634-                    continue
635-                # IE submits the full path, so trim everything but the basename.
636-                # (We can't use os.path.basename because that uses the server's
637-                # directory separator, which may not be the same as the
638-                # client's one.)
639-                filename = name_dict['filename'][name_dict['filename'].rfind("\\")+1:]
640-                FILES.appendlist(name_dict['name'], FileDict({
641-                    'filename': filename,
642-                    'content-type': 'Content-Type' in submessage and submessage['Content-Type'] or None,
643-                    'content': submessage.get_payload(),
644-                }))
645-            else:
646-                POST.appendlist(name_dict['name'], submessage.get_payload())
647-    return POST, FILES
648+    def _set_upload_handlers(self, upload_handlers):
649+        """
650+        Set the upload handler to the new handler given in the parameter.
651+        """
652+        if hasattr(self, '_files'):
653+            raise AttributeError("You cannot set the upload handler after the upload has been processed.")
654+        self._upload_handlers = upload_handlers
655 
656+    def _get_upload_handlers(self):
657+        return self._upload_handlers
658 
659+    upload_handlers = property(_get_upload_handlers, _set_upload_handlers)
660+
661+
662+    def parse_file_upload(self, META, post_data):
663+        """Returns a tuple of (POST QueryDict, FILES MultiValueDict)."""
664+        from django.core.files.fileuploadhandler import TemporaryFileUploadHandler, MemoryFileUploadHandler
665+        if not self.upload_handlers:
666+            # Order here is *very* important.
667+            self.upload_handlers = (MemoryFileUploadHandler(),
668+                                    TemporaryFileUploadHandler())
669+        else:
670+            self.upload_handlers = tuple(self.upload_handlers)
671+
672+        parser = MultiPartParser(META, post_data, self.upload_handlers,
673+                                 self.encoding)
674+        return parser.parse()
675+
676+
677 class QueryDict(MultiValueDict):
678     """
679     A specialized MultiValueDict that takes a query string when initialized.
680Index: django/test/client.py
681===================================================================
682--- django/test/client.py       (revision 7351)
683+++ django/test/client.py       (working copy)
684@@ -18,6 +18,25 @@
685 BOUNDARY = 'BoUnDaRyStRiNg'
686 MULTIPART_CONTENT = 'multipart/form-data; boundary=%s' % BOUNDARY
687 
688+class FakePayload(object):
689+    """
690+    A wrapper around StringIO that restricts what can be read,
691+    since data from the network can't be seeked and cannot
692+    be read outside of its content length (or else we hang).
693+    """
694+    def __init__(self, content):
695+        self.__content = StringIO(content)
696+        self.__len = len(content)
697+
698+    def read(self, num_bytes=None):
699+        if num_bytes is None:
700+            num_bytes = self.__len or 1
701+        assert self.__len >= num_bytes, "Cannot read more than the available bytes from the HTTP incoming data."
702+        content = self.__content.read(num_bytes)
703+        self.__len -= num_bytes
704+        return content
705+
706+
707 class ClientHandler(BaseHandler):
708     """
709     A HTTP Handler that can be used for testing purposes.
710@@ -230,7 +249,7 @@
711             'CONTENT_TYPE':   content_type,
712             'PATH_INFO':      urllib.unquote(path),
713             'REQUEST_METHOD': 'POST',
714-            'wsgi.input':     StringIO(post_data),
715+            'wsgi.input':     FakePayload(post_data),
716         }
717         r.update(extra)
718 
719Index: django/conf/global_settings.py
720===================================================================
721--- django/conf/global_settings.py      (revision 7351)
722+++ django/conf/global_settings.py      (working copy)
723@@ -224,6 +224,11 @@
724 # Example: "http://media.lawrence.com"
725 MEDIA_URL = ''
726 
727+# Directory to upload streamed files temporarily.
728+# A value of `None` means that it will use the default temporary
729+# directory for the server's operating system.
730+FILE_UPLOAD_TEMP_DIR = None
731+
732 # Default formatting for date objects. See all available format strings here:
733 # http://www.djangoproject.com/documentation/templates/#now
734 DATE_FORMAT = 'N j, Y'
735Index: django/db/models/base.py
736===================================================================
737--- django/db/models/base.py    (revision 7351)
738+++ django/db/models/base.py    (working copy)
739@@ -13,6 +13,7 @@
740 from django.utils.datastructures import SortedDict
741 from django.utils.functional import curry
742 from django.utils.encoding import smart_str, force_unicode, smart_unicode
743+from django.core.files.filemove import file_move_safe
744 from django.conf import settings
745 from itertools import izip
746 import types
747@@ -384,12 +385,16 @@
748     def _get_FIELD_size(self, field):
749         return os.path.getsize(self._get_FIELD_filename(field))
750 
751-    def _save_FIELD_file(self, field, filename, raw_contents, save=True):
752+    def _save_FIELD_file(self, field, filename, raw_field, save=True):
753         directory = field.get_directory_name()
754         try: # Create the date-based directory if it doesn't exist.
755             os.makedirs(os.path.join(settings.MEDIA_ROOT, directory))
756         except OSError: # Directory probably already exists.
757             pass
758+
759+        if filename is None:
760+            filename = raw_field.file_name
761+
762         filename = field.get_filename(filename)
763 
764         # If the filename already exists, keep adding an underscore to the name of
765@@ -406,9 +411,18 @@
766         setattr(self, field.attname, filename)
767 
768         full_filename = self._get_FIELD_filename(field)
769-        fp = open(full_filename, 'wb')
770-        fp.write(raw_contents)
771-        fp.close()
772+        if hasattr(raw_field, 'temporary_file_path'):
773+            raw_field.close()
774+            file_move_safe(raw_field.temporary_file_path(), full_filename)
775+        else:
776+            from django.core.files import filelocks
777+            fp = open(full_filename, 'wb')
778+            # exclusive lock
779+            filelocks.lock(fp, filelocks.LOCK_EX)
780+            # Stream it into the file, from where it is.
781+            for chunk in raw_field.chunk(65535):
782+                fp.write(chunk)
783+            fp.close()
784 
785         # Save the width and/or height, if applicable.
786         if isinstance(field, ImageField) and (field.width_field or field.height_field):
787Index: django/db/models/fields/__init__.py
788===================================================================
789--- django/db/models/fields/__init__.py (revision 7351)
790+++ django/db/models/fields/__init__.py (working copy)
791@@ -785,7 +785,8 @@
792         setattr(cls, 'get_%s_filename' % self.name, curry(cls._get_FIELD_filename, field=self))
793         setattr(cls, 'get_%s_url' % self.name, curry(cls._get_FIELD_url, field=self))
794         setattr(cls, 'get_%s_size' % self.name, curry(cls._get_FIELD_size, field=self))
795-        setattr(cls, 'save_%s_file' % self.name, lambda instance, filename, raw_contents, save=True: instance._save_FIELD_file(self, filename, raw_contents, save))
796+        setattr(cls, 'save_%s_file' % self.name, lambda instance, filename, raw_field, save=True: instance._save_FIELD_file(self, filename, raw_field, save))
797+        setattr(cls, 'move_%s_file' % self.name, lambda instance, raw_field, save=True: instance._save_FIELD_file(self, None, raw_field, save))
798         dispatcher.connect(self.delete_file, signal=signals.post_delete, sender=cls)
799 
800     def delete_file(self, instance):
801@@ -808,9 +809,9 @@
802         if new_data.get(upload_field_name, False):
803             func = getattr(new_object, 'save_%s_file' % self.name)
804             if rel:
805-                func(new_data[upload_field_name][0]["filename"], new_data[upload_field_name][0]["content"], save)
806+                func(new_data[upload_field_name][0].file_name, new_data[upload_field_name][0], save)
807             else:
808-                func(new_data[upload_field_name]["filename"], new_data[upload_field_name]["content"], save)
809+                func(new_data[upload_field_name].file_name, new_data[upload_field_name], save)
810 
811     def get_directory_name(self):
812         return os.path.normpath(force_unicode(datetime.datetime.now().strftime(smart_str(self.upload_to))))
813@@ -823,7 +824,7 @@
814     def save_form_data(self, instance, data):
815         from django.newforms.fields import UploadedFile
816         if data and isinstance(data, UploadedFile):
817-            getattr(instance, "save_%s_file" % self.name)(data.filename, data.content, save=False)
818+            getattr(instance, "save_%s_file" % self.name)(data.filename, data.data, save=False)
819 
820     def formfield(self, **kwargs):
821         defaults = {'form_class': forms.FileField}
822Index: django/oldforms/__init__.py
823===================================================================
824--- django/oldforms/__init__.py (revision 7351)
825+++ django/oldforms/__init__.py (working copy)
826@@ -680,18 +680,23 @@
827         self.field_name, self.is_required = field_name, is_required
828         self.validator_list = [self.isNonEmptyFile] + validator_list
829 
830-    def isNonEmptyFile(self, field_data, all_data):
831-        try:
832-            content = field_data['content']
833-        except TypeError:
834-            raise validators.CriticalValidationError, ugettext("No file was submitted. Check the encoding type on the form.")
835-        if not content:
836+    def isNonEmptyFile(self, new_data, all_data):
837+        if hasattr(new_data, 'upload_errors'):
838+            upload_errors = new_data.upload_errors()
839+            if upload_errors:
840+                raise validators.CriticalValidationError, upload_errors
841+        if not new_data.file_size:
842             raise validators.CriticalValidationError, ugettext("The submitted file is empty.")
843 
844     def render(self, data):
845         return mark_safe(u'<input type="file" id="%s" class="v%s" name="%s" />' % \
846             (self.get_id(), self.__class__.__name__, self.field_name))
847 
848+    def prepare(self, new_data):
849+        if hasattr(new_data, 'upload_errors'):
850+            upload_errors = new_data.upload_errors()
851+            new_data[self.field_name] = { '_file_upload_error': upload_errors }
852+
853     def html2python(data):
854         if data is None:
855             raise EmptyValue
856Index: django/core/handlers/wsgi.py
857===================================================================
858--- django/core/handlers/wsgi.py        (revision 7351)
859+++ django/core/handlers/wsgi.py        (working copy)
860@@ -78,6 +78,7 @@
861         self.path = force_unicode(environ['PATH_INFO'])
862         self.META = environ
863         self.method = environ['REQUEST_METHOD'].upper()
864+        self._upload_handlers = []
865 
866     def __repr__(self):
867         # Since this is called as part of error handling, we need to be very
868@@ -112,9 +113,8 @@
869         # Populates self._post and self._files
870         if self.method == 'POST':
871             if self.environ.get('CONTENT_TYPE', '').startswith('multipart'):
872-                header_dict = dict([(k, v) for k, v in self.environ.items() if k.startswith('HTTP_')])
873-                header_dict['Content-Type'] = self.environ.get('CONTENT_TYPE', '')
874-                self._post, self._files = http.parse_file_upload(header_dict, self.raw_post_data)
875+                self._raw_post_data = ''
876+                self._post, self._files = self.parse_file_upload(self.META, self.environ['wsgi.input'])
877             else:
878                 self._post, self._files = http.QueryDict(self.raw_post_data, encoding=self._encoding), datastructures.MultiValueDict()
879         else:
880Index: django/core/handlers/modpython.py
881===================================================================
882--- django/core/handlers/modpython.py   (revision 7351)
883+++ django/core/handlers/modpython.py   (working copy)
884@@ -16,6 +16,7 @@
885     def __init__(self, req):
886         self._req = req
887         self.path = force_unicode(req.uri)
888+        self._upload_handlers = []
889 
890     def __repr__(self):
891         # Since this is called as part of error handling, we need to be very
892@@ -53,7 +54,8 @@
893     def _load_post_and_files(self):
894         "Populates self._post and self._files"
895         if 'content-type' in self._req.headers_in and self._req.headers_in['content-type'].startswith('multipart'):
896-            self._post, self._files = http.parse_file_upload(self._req.headers_in, self.raw_post_data)
897+            self._raw_post_data = ''
898+            self._post, self._files = self.parse_file_upload(self.META, self._req)
899         else:
900             self._post, self._files = http.QueryDict(self.raw_post_data, encoding=self._encoding), datastructures.MultiValueDict()
901 
902Index: django/core/files/filelocks.py
903===================================================================
904--- django/core/files/filelocks.py      (revision 0)
905+++ django/core/files/filelocks.py      (revision 0)
906@@ -0,0 +1,50 @@
907+"""
908+Locking portability by Jonathan Feignberg <jdf@pobox.com> in python cookbook
909+
910+Example Usage::
911+
912+    from django.utils import file_locks
913+
914+    f = open('./file', 'wb')
915+
916+    file_locks.lock(f, file_locks.LOCK_EX)
917+    f.write('Django')
918+    f.close()
919+"""
920+
921+
922+import os
923+
924+__all__ = ['LOCK_EX','LOCK_SH','LOCK_NB','lock','unlock']
925+
926+if os.name == 'nt':
927+       import win32con
928+       import win32file
929+       import pywintypes
930+       LOCK_EX = win32con.LOCKFILE_EXCLUSIVE_LOCK
931+       LOCK_SH = 0
932+       LOCK_NB = win32con.LOCKFILE_FAIL_IMMEDIATELY
933+       __overlapped = pywintypes.OVERLAPPED()
934+elif os.name == 'posix':
935+       import fcntl
936+       LOCK_EX = fcntl.LOCK_EX
937+       LOCK_SH = fcntl.LOCK_SH
938+       LOCK_NB = fcntl.LOCK_NB
939+else:
940+       raise RuntimeError("Locking only defined for nt and posix platforms")
941+
942+if os.name == 'nt':
943+       def lock(file, flags):
944+               hfile = win32file._get_osfhandle(file.fileno())
945+               win32file.LockFileEx(hfile, flags, 0, -0x10000, __overlapped)
946+
947+       def unlock(file):
948+               hfile = win32file._get_osfhandle(file.fileno())
949+               win32file.UnlockFileEx(hfile, 0, -0x10000, __overlapped)
950+
951+elif os.name =='posix':
952+       def lock(file, flags):
953+               fcntl.flock(file.fileno(), flags)
954+
955+       def unlock(file):
956+               fcntl.flock(file.fileno(), fcntl.LOCK_UN)
957Index: django/core/files/uploadedfile.py
958===================================================================
959--- django/core/files/uploadedfile.py   (revision 0)
960+++ django/core/files/uploadedfile.py   (revision 0)
961@@ -0,0 +1,191 @@
962+"""
963+The uploaded file objects for Django.
964+This contains the base UploadedFile and the TemporaryUploadedFile
965+derived class.
966+"""
967+
968+__all__ = ('UploadedFile', 'TemporaryUploadedFile', 'InMemoryUploadedFile')
969+
970+class UploadedFile(object):
971+    """
972+    The UploadedFile object behaves somewhat like a file
973+    object and represents some data that the user submitted
974+    and is stored in some form.
975+    """
976+    DEFAULT_CHUNK_SIZE = 64 * 2**10
977+
978+    def __init__(self):
979+        self.file_size = None
980+        self.file_name = None
981+        self.content_type = None
982+        self.charset = None
983+        pass
984+
985+    def file_size(self):
986+        return self.file_size
987+
988+    def chunk(self, chunk_size=None):
989+        """
990+        Read the file to generate chunks of chunk_size bytes.
991+        """
992+        if not chunk_size:
993+            chunk_size = UploadedFile.DEFAULT_CHUNK_SIZE
994+
995+        if hasattr(self, 'seek'):
996+            self.seek(0)
997+        # Assume the pointer is at zero...
998+        counter = self.file_size()
999+
1000+        while counter > 0:
1001+            yield self.read(chunk_size)
1002+            counter -= chunk_size
1003+
1004+
1005+    def multiple_chunks(self, chunk_size=None):
1006+        """
1007+        Return True if you can expect multiple chunks, False otherwise.
1008+        Note: If a particular file representation is in memory, then
1009+              override this to return False.
1010+        """
1011+        if not chunk_size:
1012+            chunk_size = UploadedFile.DEFAULT_CHUNK_SIZE
1013+        return self.file_size() < chunk_size
1014+       
1015+
1016+    def read(self, num_bytes=None):
1017+        """
1018+        Read from the file in whatever representation it has.
1019+        """
1020+        raise NotImplementedError()
1021+
1022+    def open(self):
1023+        """
1024+        Open the file, if one needs to.
1025+        """
1026+        pass
1027+
1028+
1029+    def close(self):
1030+        """
1031+        Close the file, if one needs to.
1032+        """
1033+        pass
1034+
1035+    def __getitem__(self, key):
1036+        """
1037+        This maintains backwards compatibility.
1038+        """
1039+        import warnings
1040+        warnings.warn("The dictionary access of uploaded file objects is deprecated. Use the new object interface instead.", DeprecationWarning)
1041+        # Dictionary to translate labels
1042+        # for backwards compatbility.
1043+        # Should be removed at some point.
1044+        backwards_translate = {
1045+            'filename': 'file_name',
1046+            'content-type': 'content_type',
1047+            }
1048+
1049+        if key == 'content':
1050+            return self.read()
1051+        else:
1052+            return getattr(self, backwards_translate.get(key, key))
1053+
1054+    def __repr__(self):
1055+        """
1056+        This representation could be anything and can be overridden.
1057+        This is mostly done to make it look somewhat useful.
1058+        """
1059+        _dict = {
1060+            'file_name': self.file_name,
1061+            'content_type': self.content_type,
1062+            'content': '<omitted>',
1063+            }
1064+        return repr(_dict)
1065+
1066+
1067+class TemporaryUploadedFile(UploadedFile):
1068+    """
1069+    Upload a file to a temporary file.
1070+    """
1071+
1072+    def __init__(self, file, file_name, content_type, file_size, charset):
1073+        self.file = file
1074+        self.file_name = file_name
1075+        self.path = file.name
1076+        self.content_type = content_type
1077+        self.file_size = file_size
1078+        self.charset = charset
1079+        self.file.seek(0)
1080+
1081+    def temporary_file_path(self):
1082+        """
1083+        Return the full path of this file.
1084+        """
1085+        return self.path
1086+
1087+    def read(self, *args, **kwargs):
1088+        return self.file.read(*args, **kwargs)
1089+
1090+    def open(self):
1091+        """
1092+        Assume the person meant to seek.
1093+        """
1094+        self.seek(0)
1095+
1096+    def seek(self, *args, **kwargs):
1097+        self.file.seek(*args, **kwargs)
1098+
1099+
1100+class InMemoryUploadedFile(UploadedFile):
1101+    """
1102+    Upload a file into memory.
1103+    """
1104+    def __init__(self, file, field_name, file_name, content_type, charset):
1105+        self.file = file
1106+        self.field_name = field_name
1107+        self.file_name = file_name
1108+        self.content_type = content_type
1109+        self.charset = charset
1110+        self.file.seek(0)
1111+
1112+    def seek(self, *args, **kwargs):
1113+        self.file.seek(*args, **kwargs)
1114+
1115+    def open(self):
1116+        self.seek(0)
1117+
1118+    def read(self, *args, **kwargs):
1119+        return self.file.read(*args, **kwargs)
1120+
1121+    def chunk(self, chunk_size=None):
1122+        """
1123+        Return the entirety of the data regardless.
1124+        """
1125+        self.file.seek(0)
1126+        return self.read()
1127+
1128+    def multiple_chunks(self, chunk_size=None):
1129+        """
1130+        Since it's in memory, we'll never have multiple chunks.
1131+        """
1132+        return False
1133+
1134+
1135+class SimpleUploadedFile(InMemoryUploadedFile):
1136+    """
1137+    A simple representation of a file, which
1138+    just has content, size, and a name.
1139+    """
1140+    def __init__(self, name, content, content_type='text/plain'):
1141+        try:
1142+            from cStringIO import StringIO
1143+        except ImportError:
1144+            from StringIO import StringIO
1145+        self.file = StringIO(content or '')
1146+        self.file_name = name
1147+        self.field_name = None
1148+        self.file_size = len(content or '')
1149+        self.content_type = content_type
1150+        self.charset = None
1151+        self.file.seek(0)
1152+
1153Index: django/core/files/__init__.py
1154===================================================================
1155--- django/core/files/__init__.py       (revision 0)
1156+++ django/core/files/__init__.py       (revision 0)
1157@@ -0,0 +1 @@
1158+
1159Index: django/core/files/fileuploadhandler.py
1160===================================================================
1161--- django/core/files/fileuploadhandler.py      (revision 0)
1162+++ django/core/files/fileuploadhandler.py      (revision 0)
1163@@ -0,0 +1,243 @@
1164+""" A fileuploadhandler base and default subclass for handling file uploads.
1165+"""
1166+import os
1167+try:
1168+    from cStringIO import StringIO
1169+except ImportError:
1170+    from StringIO import StringIO
1171+
1172+from django.utils.encoding import force_unicode
1173+from django.utils.datastructures import MultiValueDict
1174+
1175+from django.core.files.uploadedfile import TemporaryUploadedFile, InMemoryUploadedFile
1176+
1177+__all__ = ('UploadFileException','StopUpload', 'SkipFile',
1178+           'FileUploadHandler', 'TemporaryFileUploadHandler',
1179+           'MemoryFileUploadHandler')
1180+
1181+
1182+class UploadFileException(Exception):
1183+    """ Any error having to do with Uploading Files. """
1184+    pass
1185+
1186+class StopUpload(UploadFileException):
1187+    """ This exception is raised when an upload must abort. """
1188+    pass
1189+
1190+class SkipFile(UploadFileException):
1191+    """ This exception is raised when a file needs to be skipped. """
1192+    pass
1193+
1194+
1195+class FileUploadHandler(object):
1196+    """ FileUploadHandler will take data and handle file uploads
1197+    in a streamed fashion.
1198+    """
1199+    chunk_size = 64 * 2 ** 10 #: The default chunk size is 64 KB.
1200+
1201+    def __init__(self):
1202+        " Initialize some local variables. "
1203+        self.file_name = None
1204+        self.content_type = None
1205+        self.content_length = None
1206+        self.charset = None
1207+
1208+    def handle_raw_input(self, input_data, META, content_length, boundary, encoding=None):
1209+        """
1210+        Handle the raw input from the client.
1211+        Parameters:
1212+          *input_data* -- An object that supports reading via .read().
1213+          *content_length* -- The (integer) value of the Content-Length header from the client.
1214+          *boundary* -- The boundary from the Content-Type header. Be sure to prepend two '--'.
1215+        """
1216+        pass
1217+
1218+    def new_file(self, field_name, file_name, content_type, content_length, charset=None):
1219+        """
1220+        Signal that a new file has been started.
1221+       
1222+        Warning: Do not trust content_length, if you get it at all.
1223+        """
1224+        self.field_name = field_name
1225+        self.file_name = file_name
1226+        self.content_type = content_type
1227+        self.content_length = content_length
1228+        self.charset = charset
1229+
1230+    def receive_data_chunk(self, raw_data, start, stop):
1231+        """
1232+        Receive data from the streamed upload parser.
1233+        Start and stop are the positions in the file.
1234+        This equality should always be true::
1235+            len(raw_data) = stop - start
1236+        """
1237+        raise NotImplementedError()
1238+
1239+    def file_complete(self, file_size):
1240+        """
1241+        Signal that a file has completed.
1242+        File size corresponds to the actual size accumulated
1243+        by all the chunks.
1244+
1245+        This should return a valid UploadedFile object.
1246+        """
1247+        raise NotImplementedError()
1248+
1249+    def upload_complete(self):
1250+        """
1251+        Signal that the upload is complete.
1252+        Do any cleanup that is necessary for this handler.
1253+        """
1254+        pass
1255+
1256+
1257+
1258+class TemporaryFileUploadHandler(FileUploadHandler):
1259+    """
1260+    Upload the streaming data into a temporary file.
1261+    """
1262+    def __init__(self, *args, **kwargs):
1263+        """ Import settings for later. """
1264+        super(TemporaryFileUploadHandler, self).__init__(*args, **kwargs)
1265+        global settings
1266+        from django.conf import settings
1267+
1268+    def new_file(self, file_name, *args, **kwargs):
1269+        """
1270+        Create the file object to append to as data is coming in.
1271+        """
1272+        super(TemporaryFileUploadHandler, self).new_file(file_name, *args, **kwargs)
1273+        self.file = TemporaryFile(settings.FILE_UPLOAD_TEMP_DIR)
1274+        self.write = self.file.write
1275+
1276+    def receive_data_chunk(self, raw_data, start, stop):
1277+        """
1278+        Once we get the data, we will save it to our file.
1279+        """
1280+        self.write(raw_data)
1281+
1282+    def file_complete(self, file_size):
1283+        """
1284+        Signal that a file has completed.
1285+        File size corresponds to the actual size accumulated
1286+        by all the chunks.
1287+
1288+        This should return a valid UploadedFile object.
1289+        """
1290+        self.file.seek(0)
1291+        return TemporaryUploadedFile(self.file, self.file_name,
1292+                                     self.content_type, file_size,
1293+                                     self.charset)
1294+
1295+
1296+class TemporaryFile(object):
1297+    """
1298+    A temporary file that tries to delete itself when garbage collected.
1299+    """
1300+    def __init__(self, dir):
1301+        import tempfile
1302+        if not dir:
1303+            dir = tempfile.gettempdir()
1304+        try:
1305+            (fd, name) = tempfile.mkstemp(suffix='.upload', dir=dir)
1306+            self.file = os.fdopen(fd, 'w+b')
1307+        except (OSError, IOError):
1308+            raise OSError, "Could not create temporary file for uploading, have you set settings.FILE_UPLOAD_TEMP_DIR correctly?"
1309+        self.name = name
1310+
1311+    def __getattr__(self, name):
1312+        a = getattr(self.__dict__['file'], name)
1313+        if type(a) != type(0):
1314+            setattr(self, name, a)
1315+        return a
1316+
1317+    def __del__(self):
1318+        try:
1319+            os.unlink(self.name)
1320+        except OSError:
1321+            pass
1322+
1323+
1324+class MemoryFileUploadHandler(FileUploadHandler):
1325+    """
1326+    The MemoryFileUploadHandler will place the data directly into memory.
1327+    """
1328+
1329+    def __init__(self):
1330+        pass
1331+
1332+
1333+    def handle_raw_input(self, input_data, META, content_length, boundary, encoding=None):
1334+        """
1335+        Parse the input data in-memory.
1336+        """
1337+        if content_length > 2621440:
1338+            # If the post is greater than 2.5 MB, do nothing.
1339+            return
1340+
1341+        from django.http import QueryDict
1342+        import email, email.Message
1343+        from cgi import parse_header
1344+
1345+        #####
1346+        # Get the headers from the META information.
1347+        headers = []
1348+        if 'HTTP_CONTENT_TYPE' not in META:
1349+            headers.append('Content-Type: %s' % (META.get('CONTENT_TYPE', '')))
1350+
1351+        if 'HTTP_CONTENT_LENGTH' not in META:
1352+            headers.append('Content-Length: %s' % (META.get('CONTENT_LENGTH', '0')))
1353+
1354+        for key, value in META.items():
1355+            if key.startswith('HTTP_'):
1356+                headers.append('%s: %s' % (key[5:].replace('_','-').title(), value))
1357+
1358+        raw_message = '\r\n'.join(headers)
1359+        raw_message += '\r\n\r\n' + input_data.read()
1360+
1361+        msg = email.message_from_string(raw_message)
1362+        POST = QueryDict('', mutable=True)
1363+        FILES = MultiValueDict()
1364+        for submessage in msg.get_payload():
1365+            if submessage and isinstance(submessage, email.Message.Message):
1366+                name_dict = parse_header(submessage['Content-Disposition'])[1]
1367+                field_name = force_unicode(name_dict['name'], encoding, errors='replace')
1368+
1369+                if 'filename' in name_dict:
1370+                    assert type([]) != type(submessage.get_payload()), "Nested MIME messages are not supported"
1371+                    if not name_dict['filename'].strip():
1372+                        continue
1373+
1374+                    filename = force_unicode(name_dict['filename'][name_dict['filename'].rfind("\\")+1:],
1375+                                             encoding, errors='replace')
1376+                    content_type = 'Content-Type' in submessage and submessage['Content-Type'] or None
1377+
1378+                    file_obj = InMemoryUploadedFile(StringIO(submessage.get_payload()),
1379+                                                    field_name, filename, content_type, None)
1380+
1381+                    FILES.appendlist(field_name, file_obj)
1382+                else:
1383+                    content = force_unicode(submessage.get_payload(), encoding, errors='replace')
1384+                    POST.appendlist(field_name, content)
1385+
1386+        return POST, FILES
1387+
1388+
1389+    def new_file(self, field_name, file_name, content_type, content_length, charset):
1390+        """
1391+        Do Nothing.
1392+        """
1393+        return
1394+
1395+    def receive_data_chunk(self, raw_data, start, stop):
1396+        """
1397+        Do nothing.
1398+        """
1399+        return
1400+
1401+    def file_complete(self, file_size):
1402+        """
1403+        Do nothing.
1404+        """
1405+        return
1406+
1407Index: django/core/files/filemove.py
1408===================================================================
1409--- django/core/files/filemove.py       (revision 0)
1410+++ django/core/files/filemove.py       (revision 0)
1411@@ -0,0 +1,52 @@
1412+import os
1413+
1414+__all__ = ('file_move_safe',)
1415+
1416+try:
1417+    import shutil
1418+    file_move = shutil.move
1419+except ImportError:
1420+    file_move = os.rename
1421+
1422+def file_move_safe(old_file_name, new_file_name, chunk_size = 1024*64, allow_overwrite=False):
1423+    """
1424+    Moves a file from one location to another in the safest way possible.
1425+   
1426+    First, it tries using shutils.move, which is OS-dependent but doesn't
1427+    break with change of filesystems. Then it tries os.rename, which will
1428+    break if it encounters a change in filesystems. Lastly, it streams
1429+    it manually from one file to another in python.
1430+
1431+    Without ``allow_overwrite``, if the destination file exists, the
1432+    file will raise an IOError.
1433+    """
1434+
1435+    from django.core.files import filelocks
1436+
1437+    if old_file_name == new_file_name:
1438+        # No file moving takes place.
1439+        return
1440+
1441+    if not allow_overwrite and os.path.exists(new_file_name):
1442+        raise IOError, "Django does not allow overwriting files."
1443+
1444+    try:
1445+        file_move(old_file_name, new_file_name)
1446+        return
1447+    except OSError: # moving to another filesystem
1448+        pass
1449+
1450+    new_file = open(new_file_name, 'wb')
1451+    # exclusive lock
1452+    filelocks.lock(new_file, filelocks.LOCK_EX)
1453+    old_file = open(old_file_name, 'rb')
1454+    current_chunk = None
1455+
1456+    while current_chunk != '':
1457+        current_chunk = old_file.read(chunk_size)
1458+        new_file.write(current_chunk)
1459+
1460+    new_file.close()
1461+    old_file.close()
1462+
1463+    os.remove(old_file_name)
1464Index: django/newforms/fields.py
1465===================================================================
1466--- django/newforms/fields.py   (revision 7351)
1467+++ django/newforms/fields.py   (working copy)
1468@@ -416,9 +416,9 @@
1469 
1470 class UploadedFile(StrAndUnicode):
1471     "A wrapper for files uploaded in a FileField"
1472-    def __init__(self, filename, content):
1473+    def __init__(self, filename, data):
1474         self.filename = filename
1475-        self.content = content
1476+        self.data = data
1477 
1478     def __unicode__(self):
1479         """
1480@@ -445,12 +445,10 @@
1481         elif not data and initial:
1482             return initial
1483         try:
1484-            f = UploadedFile(data['filename'], data['content'])
1485-        except TypeError:
1486+            f = UploadedFile(data.file_name, data)
1487+        except (TypeError, AttributeError):
1488             raise ValidationError(self.error_messages['invalid'])
1489-        except KeyError:
1490-            raise ValidationError(self.error_messages['missing'])
1491-        if not f.content:
1492+        if not f.data.file_size:
1493             raise ValidationError(self.error_messages['empty'])
1494         return f
1495 
1496@@ -470,15 +468,26 @@
1497         elif not data and initial:
1498             return initial
1499         from PIL import Image
1500-        from cStringIO import StringIO
1501+
1502+        # We need to get the file, it either has a path
1503+        # or we have to read it all into memory...
1504+        if hasattr(data, 'temporary_file_path'):
1505+            file = data.temporary_file_path()
1506+        else:
1507+            try:
1508+                from cStringIO import StringIO
1509+            except ImportError:
1510+                from StringIO import StringIO
1511+            file = StringIO(data.read())
1512+
1513         try:
1514             # load() is the only method that can spot a truncated JPEG,
1515             #  but it cannot be called sanely after verify()
1516-            trial_image = Image.open(StringIO(f.content))
1517+            trial_image = Image.open(file)
1518             trial_image.load()
1519             # verify() is the only method that can spot a corrupt PNG,
1520             #  but it must be called immediately after the constructor
1521-            trial_image = Image.open(StringIO(f.content))
1522+            trial_image = Image.open(file)
1523             trial_image.verify()
1524         except Exception: # Python Imaging Library doesn't recognize it as an image
1525             raise ValidationError(self.error_messages['invalid_image'])
1526Index: tests/modeltests/model_forms/models.py
1527===================================================================
1528--- tests/modeltests/model_forms/models.py      (revision 7351)
1529+++ tests/modeltests/model_forms/models.py      (working copy)
1530@@ -75,6 +75,7 @@
1531 __test__ = {'API_TESTS': """
1532 >>> from django import newforms as forms
1533 >>> from django.newforms.models import ModelForm
1534+>>> from django.core.files.uploadedfile import SimpleUploadedFile
1535 
1536 The bare bones, absolutely nothing custom, basic case.
1537 
1538@@ -792,7 +793,7 @@
1539 
1540 # Upload a file and ensure it all works as expected.
1541 
1542->>> f = TextFileForm(data={'description': u'Assistance'}, files={'file': {'filename': 'test1.txt', 'content': 'hello world'}})
1543+>>> f = TextFileForm(data={'description': u'Assistance'}, files={'file': SimpleUploadedFile('test1.txt', 'hello world')})
1544 >>> f.is_valid()
1545 True
1546 >>> type(f.cleaned_data['file'])
1547@@ -819,7 +820,7 @@
1548 
1549 # Override the file by uploading a new one.
1550 
1551->>> f = TextFileForm(data={'description': u'Assistance'}, files={'file': {'filename': 'test2.txt', 'content': 'hello world'}}, instance=instance)
1552+>>> f = TextFileForm(data={'description': u'Assistance'}, files={'file': SimpleUploadedFile('test2.txt', 'hello world')}, instance=instance)
1553 >>> f.is_valid()
1554 True
1555 >>> instance = f.save()
1556@@ -838,7 +839,7 @@
1557 >>> instance.file
1558 ''
1559 
1560->>> f = TextFileForm(data={'description': u'Assistance'}, files={'file': {'filename': 'test3.txt', 'content': 'hello world'}}, instance=instance)
1561+>>> f = TextFileForm(data={'description': u'Assistance'}, files={'file': SimpleUploadedFile('test3.txt', 'hello world')}, instance=instance)
1562 >>> f.is_valid()
1563 True
1564 >>> instance = f.save()
1565@@ -858,7 +859,7 @@
1566 
1567 >>> image_data = open(os.path.join(os.path.dirname(__file__), "test.png")).read()
1568 
1569->>> f = ImageFileForm(data={'description': u'An image'}, files={'image': {'filename': 'test.png', 'content': image_data}})
1570+>>> f = ImageFileForm(data={'description': u'An image'}, files={'image': SimpleUploadedFile('test.png', image_data)})
1571 >>> f.is_valid()
1572 True
1573 >>> type(f.cleaned_data['image'])
1574@@ -885,7 +886,7 @@
1575 
1576 # Override the file by uploading a new one.
1577 
1578->>> f = ImageFileForm(data={'description': u'Changed it'}, files={'image': {'filename': 'test2.png', 'content': image_data}}, instance=instance)
1579+>>> f = ImageFileForm(data={'description': u'Changed it'}, files={'image': SimpleUploadedFile('test2.png', image_data)}, instance=instance)
1580 >>> f.is_valid()
1581 True
1582 >>> instance = f.save()
1583@@ -904,7 +905,7 @@
1584 >>> instance.image
1585 ''
1586 
1587->>> f = ImageFileForm(data={'description': u'And a final one'}, files={'image': {'filename': 'test3.png', 'content': image_data}}, instance=instance)
1588+>>> f = ImageFileForm(data={'description': u'And a final one'}, files={'image': SimpleUploadedFile('test3.png', image_data)}, instance=instance)
1589 >>> f.is_valid()
1590 True
1591 >>> instance = f.save()
1592Index: tests/regressiontests/bug639/tests.py
1593===================================================================
1594--- tests/regressiontests/bug639/tests.py       (revision 7351)
1595+++ tests/regressiontests/bug639/tests.py       (working copy)
1596@@ -9,6 +9,7 @@
1597 from regressiontests.bug639.models import Photo
1598 from django.http import QueryDict
1599 from django.utils.datastructures import MultiValueDict
1600+from django.core.files.uploadedfile import SimpleUploadedFile
1601 
1602 class Bug639Test(unittest.TestCase):
1603         
1604@@ -21,11 +22,7 @@
1605         
1606         # Fake a request query dict with the file
1607         qd = QueryDict("title=Testing&image=", mutable=True)
1608-        qd["image_file"] = {
1609-            "filename" : "test.jpg",
1610-            "content-type" : "image/jpeg",
1611-            "content" : img
1612-        }
1613+        qd["image_file"] = SimpleUploadedFile('test.jpg', img, 'image/jpeg')
1614         
1615         manip = Photo.AddManipulator()
1616         manip.do_html2python(qd)
1617@@ -39,4 +36,4 @@
1618         Make sure to delete the "uploaded" file to avoid clogging /tmp.
1619         """
1620         p = Photo.objects.get()
1621-        os.unlink(p.get_image_filename())
1622\ No newline at end of file
1623+        os.unlink(p.get_image_filename())
1624Index: tests/regressiontests/forms/error_messages.py
1625===================================================================
1626--- tests/regressiontests/forms/error_messages.py       (revision 7351)
1627+++ tests/regressiontests/forms/error_messages.py       (working copy)
1628@@ -1,6 +1,7 @@
1629 # -*- coding: utf-8 -*-
1630 tests = r"""
1631 >>> from django.newforms import *
1632+>>> from django.core.files.uploadedfile import SimpleUploadedFile
1633 
1634 # CharField ###################################################################
1635 
1636@@ -214,11 +215,11 @@
1637 Traceback (most recent call last):
1638 ...
1639 ValidationError: [u'INVALID']
1640->>> f.clean({})
1641+>>> f.clean(SimpleUploadedFile('name', None))
1642 Traceback (most recent call last):
1643 ...
1644-ValidationError: [u'MISSING']
1645->>> f.clean({'filename': 'name', 'content':''})
1646+ValidationError: [u'EMPTY FILE']
1647+>>> f.clean(SimpleUploadedFile('name', ''))
1648 Traceback (most recent call last):
1649 ...
1650 ValidationError: [u'EMPTY FILE']
1651Index: tests/regressiontests/forms/fields.py
1652===================================================================
1653--- tests/regressiontests/forms/fields.py       (revision 7351)
1654+++ tests/regressiontests/forms/fields.py       (working copy)
1655@@ -2,6 +2,7 @@
1656 tests = r"""
1657 >>> from django.newforms import *
1658 >>> from django.newforms.widgets import RadioFieldRenderer
1659+>>> from django.core.files.uploadedfile import SimpleUploadedFile
1660 >>> import datetime
1661 >>> import time
1662 >>> import re
1663@@ -773,12 +774,12 @@
1664 >>> f.clean({})
1665 Traceback (most recent call last):
1666 ...
1667-ValidationError: [u'No file was submitted.']
1668+ValidationError: [u'No file was submitted. Check the encoding type on the form.']
1669 
1670 >>> f.clean({}, '')
1671 Traceback (most recent call last):
1672 ...
1673-ValidationError: [u'No file was submitted.']
1674+ValidationError: [u'No file was submitted. Check the encoding type on the form.']
1675 
1676 >>> f.clean({}, 'files/test3.pdf')
1677 'files/test3.pdf'
1678@@ -788,20 +789,20 @@
1679 ...
1680 ValidationError: [u'No file was submitted. Check the encoding type on the form.']
1681 
1682->>> f.clean({'filename': 'name', 'content': None})
1683+>>> f.clean(SimpleUploadedFile('name', None))
1684 Traceback (most recent call last):
1685 ...
1686 ValidationError: [u'The submitted file is empty.']
1687 
1688->>> f.clean({'filename': 'name', 'content': ''})
1689+>>> f.clean(SimpleUploadedFile('name', ''))
1690 Traceback (most recent call last):
1691 ...
1692 ValidationError: [u'The submitted file is empty.']
1693 
1694->>> type(f.clean({'filename': 'name', 'content': 'Some File Content'}))
1695+>>> type(f.clean(SimpleUploadedFile('name', 'Some File Content')))
1696 <class 'django.newforms.fields.UploadedFile'>
1697 
1698->>> type(f.clean({'filename': 'name', 'content': 'Some File Content'}, 'files/test4.pdf'))
1699+>>> type(f.clean(SimpleUploadedFile('name', 'Some File Content'), 'files/test4.pdf'))
1700 <class 'django.newforms.fields.UploadedFile'>
1701 
1702 # URLField ##################################################################
1703Index: tests/regressiontests/forms/forms.py
1704===================================================================
1705--- tests/regressiontests/forms/forms.py        (revision 7351)
1706+++ tests/regressiontests/forms/forms.py        (working copy)
1707@@ -1,6 +1,7 @@
1708 # -*- coding: utf-8 -*-
1709 tests = r"""
1710 >>> from django.newforms import *
1711+>>> from django.core.files.uploadedfile import SimpleUploadedFile
1712 >>> import datetime
1713 >>> import time
1714 >>> import re
1715@@ -1465,7 +1466,7 @@
1716 >>> print f
1717 <tr><th>File1:</th><td><ul class="errorlist"><li>This field is required.</li></ul><input type="file" name="file1" /></td></tr>
1718 
1719->>> f = FileForm(data={}, files={'file1': {'filename': 'name', 'content':''}}, auto_id=False)
1720+>>> f = FileForm(data={}, files={'file1': SimpleUploadedFile('name', '')}, auto_id=False)
1721 >>> print f
1722 <tr><th>File1:</th><td><ul class="errorlist"><li>The submitted file is empty.</li></ul><input type="file" name="file1" /></td></tr>
1723 
1724@@ -1473,7 +1474,7 @@
1725 >>> print f
1726 <tr><th>File1:</th><td><ul class="errorlist"><li>No file was submitted. Check the encoding type on the form.</li></ul><input type="file" name="file1" /></td></tr>
1727 
1728->>> f = FileForm(data={}, files={'file1': {'filename': 'name', 'content':'some content'}}, auto_id=False)
1729+>>> f = FileForm(data={}, files={'file1': SimpleUploadedFile('name', 'some content')}, auto_id=False)
1730 >>> print f
1731 <tr><th>File1:</th><td><input type="file" name="file1" /></td></tr>
1732 >>> f.is_valid()
1733Index: tests/regressiontests/test_client_regress/views.py
1734===================================================================
1735--- tests/regressiontests/test_client_regress/views.py  (revision 7351)
1736+++ tests/regressiontests/test_client_regress/views.py  (working copy)
1737@@ -1,5 +1,6 @@
1738 from django.contrib.auth.decorators import login_required
1739 from django.http import HttpResponse, HttpResponseRedirect, HttpResponseServerError
1740+import sha
1741 
1742 def no_template_view(request):
1743     "A simple view that expects a GET request, and returns a rendered template"
1744@@ -10,13 +11,36 @@
1745     Check that a file upload can be updated into the POST dictionary without
1746     going pear-shaped.
1747     """
1748+    from django.core.files.uploadedfile import UploadedFile
1749     form_data = request.POST.copy()
1750     form_data.update(request.FILES)
1751-    if isinstance(form_data['file_field'], dict) and isinstance(form_data['name'], unicode):
1752+    if isinstance(form_data['file_field'], UploadedFile) and isinstance(form_data['name'], unicode):
1753         return HttpResponse('')
1754     else:
1755         return HttpResponseServerError()
1756 
1757+def file_upload_view_verify(request):
1758+    """
1759+    Use the sha digest hash to verify the uploaded contents.
1760+    """
1761+    from django.core.files.uploadedfile import UploadedFile
1762+    form_data = request.POST.copy()
1763+    form_data.update(request.FILES)
1764+    for key, value in form_data.items():
1765+        if key.endswith('_hash'):
1766+            continue
1767+        if key + '_hash' not in form_data:
1768+            continue
1769+        submitted_hash = form_data[key + '_hash']
1770+        if isinstance(value, UploadedFile):
1771+            new_hash = sha.new(value.read()).hexdigest()
1772+        else:
1773+            new_hash = sha.new(value).hexdigest()
1774+        if new_hash != submitted_hash:
1775+            return HttpResponseServerError()
1776+
1777+    return HttpResponse('')
1778+
1779 def get_view(request):
1780     "A simple login protected view"
1781     return HttpResponse("Hello world")
1782@@ -37,4 +61,4 @@
1783 def login_protected_redirect_view(request):
1784     "A view that redirects all requests to the GET view"
1785     return HttpResponseRedirect('/test_client_regress/get_view/')
1786-login_protected_redirect_view = login_required(login_protected_redirect_view)
1787\ No newline at end of file
1788+login_protected_redirect_view = login_required(login_protected_redirect_view)
1789Index: tests/regressiontests/test_client_regress/models.py
1790===================================================================
1791--- tests/regressiontests/test_client_regress/models.py (revision 7351)
1792+++ tests/regressiontests/test_client_regress/models.py (working copy)
1793@@ -5,6 +5,7 @@
1794 from django.test import Client, TestCase
1795 from django.core.urlresolvers import reverse
1796 import os
1797+import sha
1798 
1799 class AssertContainsTests(TestCase):
1800     def test_contains(self):
1801@@ -243,6 +244,44 @@
1802         response = self.client.post('/test_client_regress/file_upload/', post_data)
1803         self.assertEqual(response.status_code, 200)
1804 
1805+    def test_large_upload(self):
1806+        import tempfile
1807+        dir = tempfile.gettempdir()
1808+
1809+        (fd, name1) = tempfile.mkstemp(suffix='.file1', dir=dir)
1810+        file1 = os.fdopen(fd, 'w+b')
1811+        file1.write('a' * (2 ** 21))
1812+        file1.seek(0)
1813+
1814+        (fd, name2) = tempfile.mkstemp(suffix='.file2', dir=dir)
1815+        file2 = os.fdopen(fd, 'w+b')
1816+        file2.write('a' * (10 * 2 ** 20))
1817+        file2.seek(0)
1818+
1819+        post_data = {
1820+            'name': 'Ringo',
1821+            'file_field1': file1,
1822+            'file_field2': file2,
1823+            }
1824+
1825+        for key in post_data.keys():
1826+            try:
1827+                post_data[key + '_hash'] = sha.new(post_data[key].read()).hexdigest()
1828+                post_data[key].seek(0)
1829+            except AttributeError:
1830+                post_data[key + '_hash'] = sha.new(post_data[key]).hexdigest()
1831+
1832+        response = self.client.post('/test_client_regress/file_upload_verify/', post_data)
1833+
1834+        for name in (name1, name2):
1835+            try:
1836+                os.unlink(name)
1837+            except:
1838+                pass
1839+
1840+        self.assertEqual(response.status_code, 200)
1841+
1842+
1843 class LoginTests(TestCase):
1844     fixtures = ['testdata']
1845 
1846Index: tests/regressiontests/test_client_regress/urls.py
1847===================================================================
1848--- tests/regressiontests/test_client_regress/urls.py   (revision 7351)
1849+++ tests/regressiontests/test_client_regress/urls.py   (working copy)
1850@@ -4,6 +4,7 @@
1851 urlpatterns = patterns('',
1852     (r'^no_template_view/$', views.no_template_view),
1853     (r'^file_upload/$', views.file_upload_view),
1854+    (r'^file_upload_verify/$', views.file_upload_view_verify),
1855     (r'^get_view/$', views.get_view),
1856     url(r'^arg_view/(?P<name>.+)/$', views.view_with_argument, name='arg_view'),
1857     (r'^login_protected_redirect_view/$', views.login_protected_redirect_view)
1858Index: docs/settings.txt
1859===================================================================
1860--- docs/settings.txt   (revision 7351)
1861+++ docs/settings.txt   (working copy)
1862@@ -513,6 +513,15 @@
1863 The character encoding used to decode any files read from disk. This includes
1864 template files and initial SQL data files.
1865 
1866+FILE_UPLOAD_TEMP_DIR
1867+--------------------
1868+
1869+**New in Django development version**
1870+
1871+Default: ``None``
1872+
1873+The directory to store data temporarily while uploading files. If ``None``, Django will use the standard temporary directory for the operating system. For example, this will default to '/tmp' on *nix-style operating systems.
1874+
1875 FIXTURE_DIRS
1876 -------------
1877