Code

Ticket #2070: 5100_upload_request_meta.diff

File 5100_upload_request_meta.diff, 33.8 KB (added by Michael Axiak <axiak@…>, 7 years ago)

This isn't important enough to get its own property in request...request.META now contains 'UPLOAD_PROGRESS_ID'

Line 
1Index: django/http/__init__.py
2===================================================================
3--- django/http/__init__.py     (revision 5100)
4+++ django/http/__init__.py     (working copy)
5@@ -1,11 +1,18 @@
6-import os
7+import os, pickle
8 from Cookie import SimpleCookie
9 from pprint import pformat
10 from urllib import urlencode, quote
11 from django.utils.datastructures import MultiValueDict
12+import re
13 
14+try:
15+    from cStringIO import StringIO
16+except ImportError:
17+    from StringIO import StringIO
18+
19 RESERVED_CHARS="!*'();:@&=+$,/?%#[]"
20 
21+
22 try:
23     # The mod_python version is more efficient, so try importing it first.
24     from mod_python.util import parse_qsl
25@@ -17,6 +24,10 @@
26 
27 class HttpRequest(object):
28     "A basic HTTP request"
29+
30+    upload_id_re = re.compile(r'[a-fA-F0-9]{32}')
31+
32+
33     def __init__(self):
34         self.GET, self.POST, self.COOKIES, self.META, self.FILES = {}, {}, {}, {}, {}
35         self.path = ''
36@@ -42,37 +53,334 @@
37     def is_secure(self):
38         return os.environ.get("HTTPS") == "on"
39 
40-def parse_file_upload(header_dict, post_data):
41-    "Returns a tuple of (POST MultiValueDict, FILES MultiValueDict)"
42-    import email, email.Message
43-    from cgi import parse_header
44-    raw_message = '\r\n'.join(['%s:%s' % pair for pair in header_dict.items()])
45-    raw_message += '\r\n\r\n' + post_data
46-    msg = email.message_from_string(raw_message)
47-    POST = MultiValueDict()
48-    FILES = MultiValueDict()
49-    for submessage in msg.get_payload():
50-        if submessage and isinstance(submessage, email.Message.Message):
51-            name_dict = parse_header(submessage['Content-Disposition'])[1]
52-            # name_dict is something like {'name': 'file', 'filename': 'test.txt'} for file uploads
53-            # or {'name': 'blah'} for POST fields
54-            # We assume all uploaded files have a 'filename' set.
55-            if 'filename' in name_dict:
56-                assert type([]) != type(submessage.get_payload()), "Nested MIME messages are not supported"
57-                if not name_dict['filename'].strip():
58-                    continue
59-                # IE submits the full path, so trim everything but the basename.
60-                # (We can't use os.path.basename because it expects Linux paths.)
61-                filename = name_dict['filename'][name_dict['filename'].rfind("\\")+1:]
62-                FILES.appendlist(name_dict['name'], {
63-                    'filename': filename,
64-                    'content-type': 'Content-Type' in submessage and submessage['Content-Type'] or None,
65-                    'content': submessage.get_payload(),
66-                })
67+    def _get_file_progress_from_args(self, headers, get, querystring):
68+
69+        if 'X-Upload-ID' in headers:
70+            progress_id = headers['X-Upload-ID']
71+        elif 'X-Progress-ID' in headers:
72+            progress_id = headers['X-Progress-ID']
73+        elif 'upload_id' in get:
74+            progress_id = get['upload_id']
75+        elif 'progress_id' in get:
76+            progress_id = get['progress_id']
77+        elif querystring != None and len(querystring.strip()) == 32:
78+            progress_id = querystring
79+        else:
80+            return None
81+
82+        if not self.upload_id_re.match(progress_id):
83+            return None
84+
85+        return progress_id
86+
87+
88+def parse_file_upload(headers, input, progress_id = None):
89+    from django.conf import settings
90+
91+    # Only stream files to disk if FILE_STREAMING_DIR is set
92+    file_upload_dir = settings.FILE_UPLOAD_DIR
93+    streaming_min_post_size = settings.STREAMING_MIN_POST_SIZE
94+
95+    try:
96+        parser = MultiPartParser(headers, input, progress_id, file_upload_dir, streaming_min_post_size)
97+        return parser.parse()
98+    except MultiPartParserError, e:
99+        return MultiValueDict({ '_file_upload_error': [e.message] }), {}
100+
101+class MultiPartParserError(Exception):
102+    def __init__(self, message):
103+        self.message = message
104+    def __str__(self):
105+        return repr(self.message)
106+       
107+class MultiPartParser(object):
108+    """
109+    A rfc2388 multipart/form-data parser.
110+   
111+    parse() reads the input stream in chunk_size chunks and returns a
112+    tuple of (POST MultiValueDict, FILES MultiValueDict). If
113+    file_upload_dir is defined files will be streamed to temporary
114+    files in the specified directory.
115+
116+    The FILES dictionary will have 'filename', 'content-type',
117+    'content' and 'content-length' entries. For streamed files it will
118+    also have 'tmpfilename' and 'tmpfile'. The 'content' entry will
119+    only be read from disk when referenced for streamed files.
120+
121+    If the header X-Progress-ID is sent with a 32 character hex string
122+    a temporary file with the same name will be created in
123+    `file_upload_dir`` with a pickled { 'received', 'size' }
124+    dictionary with the number of bytes received and the size expected
125+    respectively. The file will be unlinked when the parser finishes.
126+
127+    """
128+
129+    def __init__(self, headers, input, progress_id=None, file_upload_dir=None, streaming_min_post_size=None, chunk_size=1024*64):
130+        try:
131+            content_length = int(headers['Content-Length'])
132+        except:
133+            raise MultiPartParserError('Invalid Content-Length: %s' % headers.get('Content-Length'))
134+
135+        content_type = headers.get('Content-Type')
136+
137+        if not content_type or not content_type.startswith('multipart/'):
138+            raise MultiPartParserError('Invalid Content-Type: %s' % content_type)
139+           
140+        ctype, opts = self.parse_header(content_type)
141+        boundary = opts.get('boundary')
142+        from cgi import valid_boundary
143+        if not boundary or not valid_boundary(boundary):
144+            raise MultiPartParserError('Invalid boundary in multipart form: %s' % boundary)
145+
146+        # check if we got a valid X-Progress-ID id
147+        if file_upload_dir and progress_id:
148+            import re
149+            if re.match(r'^[0-9a-zA-Z]{32}$', progress_id):
150+                self._progress_filename = os.path.join(file_upload_dir, progress_id)
151             else:
152-                POST.appendlist(name_dict['name'], submessage.get_payload())
153-    return POST, FILES
154+                raise MultiPartParserError('Invalid X-Progress-ID: %s' % progress_id)
155+        else:
156+            self._progress_filename = None
157+        self._boundary = '--' + boundary
158+        self._input = input
159+        self._size = content_length
160+        self._received = 0
161+        self._file_upload_dir = file_upload_dir
162+        self._chunk_size = chunk_size
163+        self._state = 'PREAMBLE'
164+        self._partial = ''
165+        self._post = MultiValueDict()
166+        self._files = MultiValueDict()
167+        if streaming_min_post_size is not None and content_length < streaming_min_post_size:
168+            self._file_upload_dir = None # disable file streaming for small request
169 
170+        try:
171+            # use mx fast string search if available
172+            from mx.TextTools import FS
173+            self._fs = FS(self._boundary)
174+        except ImportError:
175+            self._fs = None
176+
177+    def parse(self):
178+        try:
179+            self._parse()
180+        finally:
181+            if self._progress_filename:
182+                try:
183+                    os.unlink(self._progress_filename)
184+                except OSError:
185+                    pass
186+       
187+        return self._post, self._files
188+
189+    def _parse(self):
190+        size = self._size
191+
192+        try:
193+            while size > 0:
194+                n = self._read(self._input, min(self._chunk_size, size))
195+                if not n:
196+                    break
197+                size -= n
198+        except:
199+            # consume any remaining data so we dont generate a "Connection Reset" error
200+            size = self._size - self._received
201+            while size > 0:
202+                data = self._input.read(min(self._chunk_size, size))
203+                size -= len(data)
204+            raise
205+
206+    def _find_boundary(self, data, start, stop):
207+        """
208+        Find the next boundary and return the end of current part
209+        and start of next part.
210+        """
211+        if self._fs:
212+            boundary = self._fs.find(data, start, stop)
213+        else:
214+            boundary = data.find(self._boundary, start, stop)
215+        if boundary >= 0:
216+            end = boundary
217+            next = boundary + len(self._boundary)
218+
219+            # backup over CRLF
220+            if end > 0 and data[end-1] == '\n': end -= 1
221+            if end > 0 and data[end-1] == '\r': end -= 1
222+            # skip over --CRLF
223+            if next < stop and data[next] == '-': next += 1
224+            if next < stop and data[next] == '-': next += 1
225+            if next < stop and data[next] == '\r': next += 1
226+            if next < stop and data[next] == '\n': next += 1
227+
228+            return True, end, next
229+        else:
230+            return False, stop, stop
231+
232+    class TemporaryFile(object):
233+        "A temporary file that tries to delete itself when garbage collected."
234+        def __init__(self, dir):
235+            import tempfile
236+            (fd, name) = tempfile.mkstemp(suffix='.upload', dir=dir)
237+            self.file = os.fdopen(fd, 'w+b')
238+            self.name = name
239+
240+        def __getattr__(self, name):
241+            a = getattr(self.__dict__['file'], name)
242+            if type(a) != type(0):
243+                setattr(self, name, a)
244+            return a
245+
246+        def __del__(self):
247+            try:
248+                os.unlink(self.name)
249+            except OSError:
250+                pass
251+           
252+    class LazyContent(dict):
253+        """
254+        A lazy FILES dictionary entry that reads the contents from
255+        tmpfile only when referenced.
256+        """
257+        def __init__(self, data):
258+            dict.__init__(self, data)
259+       
260+        def __getitem__(self, key):
261+            if key == 'content' and not self.has_key(key):
262+                self['tmpfile'].seek(0)
263+                self['content'] = self['tmpfile'].read()
264+            return dict.__getitem__(self, key)
265+
266+    def _read(self, input, size):
267+        data = input.read(size)
268+
269+        if not data:
270+            return 0
271+
272+        read_size = len(data)
273+        self._received += read_size
274+
275+        if self._partial:
276+            data = self._partial + data
277+
278+        start = 0
279+        stop = len(data)
280+       
281+        while start < stop:
282+            boundary, end, next = self._find_boundary(data, start, stop)
283+
284+            if not boundary and read_size:
285+                # make sure we dont treat a partial boundary (and its separators) as data
286+                stop -= len(self._boundary) + 16
287+                end = next = stop
288+                if end <= start:
289+                    break # need more data
290+
291+            if self._state == 'PREAMBLE':
292+                # Preamble, just ignore it
293+                self._state = 'HEADER'
294+
295+            elif self._state == 'HEADER':
296+                # Beginning of header, look for end of header and parse it if found.
297+
298+                header_end = data.find('\r\n\r\n', start, stop)
299+                if header_end == -1:
300+                    break # need more data
301+
302+                header = data[start:header_end]
303+
304+                self._fieldname = None
305+                self._filename = None
306+                self._content_type = None
307+
308+                for line in header.split('\r\n'):
309+                    ctype, opts = self.parse_header(line)
310+                    if ctype == 'content-disposition: form-data':
311+                        self._fieldname = opts.get('name')
312+                        self._filename = opts.get('filename')
313+                    elif ctype.startswith('content-type: '):
314+                        self._content_type = ctype[14:]
315+
316+                if self._filename is not None:
317+                    # cleanup filename from IE full paths:
318+                    self._filename = self._filename[self._filename.rfind("\\")+1:].strip()
319+
320+                    if self._filename: # ignore files without filenames
321+                        if self._file_upload_dir:
322+                            try:
323+                                self._file = self.TemporaryFile(dir=self._file_upload_dir)
324+                            except:
325+                                raise MultiPartParserError("Failed to create temporary file.")
326+                        else:
327+                            self._file = StringIO()
328+                    else:
329+                        self._file = None
330+                    self._filesize = 0
331+                    self._state = 'FILE'
332+                else:
333+                    self._field = StringIO()
334+                    self._state = 'FIELD'
335+                next = header_end + 4
336+
337+            elif self._state == 'FIELD':
338+                # In a field, collect data until a boundary is found.
339+
340+                self._field.write(data[start:end])
341+                if boundary:
342+                    if self._fieldname:
343+                        self._post.appendlist(self._fieldname, self._field.getvalue())
344+                    self._field.close()
345+                    self._state = 'HEADER'
346+
347+            elif self._state == 'FILE':
348+                # In a file, collect data until a boundary is found.
349+
350+                if self._file:
351+                    try:
352+                        self._file.write(data[start:end])
353+                    except IOError, e:
354+                        raise MultiPartParserError("Failed to write to temporary file.")
355+                    self._filesize += end-start
356+
357+                    if self._progress_filename:
358+                        f = open(os.path.join(self._file_upload_dir, self._progress_filename), 'w')
359+                        pickle.dump({ 'received': self._received, 'size': self._size }, f)
360+                        f.close()
361+
362+                if boundary:
363+                    if self._file:
364+                        if self._file_upload_dir:
365+                            self._file.seek(0)
366+                            file = self.LazyContent({
367+                                'filename': self._filename,
368+                                'content-type':  self._content_type,
369+                                # 'content': is read on demand
370+                                'content-length': self._filesize,
371+                                'tmpfilename': self._file.name,
372+                                'tmpfile': self._file
373+                            })
374+                        else:
375+                            file = {
376+                                'filename': self._filename,
377+                                'content-type':  self._content_type,
378+                                'content': self._file.getvalue(),
379+                                'content-length': self._filesize
380+                            }
381+                            self._file.close()
382+
383+                        self._files.appendlist(self._fieldname, file)
384+
385+                    self._state = 'HEADER'
386+
387+            start = next
388+               
389+        self._partial = data[start:]
390+
391+        return read_size
392+
393+    def parse_header(self, line):
394+        from cgi import parse_header
395+        return parse_header(line)
396+
397 class QueryDict(MultiValueDict):
398     """A specialized MultiValueDict that takes a query string when initialized.
399     This is immutable unless you create a copy of it."""
400@@ -306,3 +614,4 @@
401     if not host:
402         host = request.META.get('HTTP_HOST', '')
403     return host
404+
405Index: django/oldforms/__init__.py
406===================================================================
407--- django/oldforms/__init__.py (revision 5100)
408+++ django/oldforms/__init__.py (working copy)
409@@ -666,17 +666,22 @@
410         self.validator_list = [self.isNonEmptyFile] + validator_list
411 
412     def isNonEmptyFile(self, field_data, all_data):
413-        try:
414-            content = field_data['content']
415-        except TypeError:
416+        if field_data.has_key('_file_upload_error'):
417+            raise validators.CriticalValidationError, field_data['_file_upload_error']
418+        if not field_data.has_key('filename'):
419             raise validators.CriticalValidationError, gettext("No file was submitted. Check the encoding type on the form.")
420-        if not content:
421+        if not field_data['content-length']:
422             raise validators.CriticalValidationError, gettext("The submitted file is empty.")
423 
424     def render(self, data):
425         return '<input type="file" id="%s" class="v%s" name="%s" />' % \
426             (self.get_id(), self.__class__.__name__, self.field_name)
427 
428+    def prepare(self, new_data):
429+        if new_data.has_key('_file_upload_error'):
430+            # pretend we got something in the field to raise a validation error later
431+            new_data[self.field_name] = { '_file_upload_error': new_data['_file_upload_error'] }
432+
433     def html2python(data):
434         if data is None:
435             raise EmptyValue
436Index: django/db/models/base.py
437===================================================================
438--- django/db/models/base.py    (revision 5100)
439+++ django/db/models/base.py    (working copy)
440@@ -12,12 +12,14 @@
441 from django.dispatch import dispatcher
442 from django.utils.datastructures import SortedDict
443 from django.utils.functional import curry
444+from django.utils.file import file_move_safe
445 from django.conf import settings
446 from itertools import izip
447 import types
448 import sys
449 import os
450 
451+               
452 class ModelBase(type):
453     "Metaclass for all models"
454     def __new__(cls, name, bases, attrs):
455@@ -361,7 +363,7 @@
456     def _get_FIELD_size(self, field):
457         return os.path.getsize(self._get_FIELD_filename(field))
458 
459-    def _save_FIELD_file(self, field, filename, raw_contents, save=True):
460+    def _save_FIELD_file(self, field, filename, raw_field, save=True):
461         directory = field.get_directory_name()
462         try: # Create the date-based directory if it doesn't exist.
463             os.makedirs(os.path.join(settings.MEDIA_ROOT, directory))
464@@ -383,9 +385,13 @@
465         setattr(self, field.attname, filename)
466 
467         full_filename = self._get_FIELD_filename(field)
468-        fp = open(full_filename, 'wb')
469-        fp.write(raw_contents)
470-        fp.close()
471+        if raw_field.has_key('tmpfilename'):
472+            raw_field['tmpfile'].close()
473+            file_move_safe(raw_field['tmpfilename'], full_filename)
474+        else:
475+            fp = open(full_filename, 'wb')
476+            fp.write(raw_field['content'])
477+            fp.close()
478 
479         # Save the width and/or height, if applicable.
480         if isinstance(field, ImageField) and (field.width_field or field.height_field):
481Index: django/db/models/fields/__init__.py
482===================================================================
483--- django/db/models/fields/__init__.py (revision 5100)
484+++ django/db/models/fields/__init__.py (working copy)
485@@ -636,7 +636,7 @@
486         setattr(cls, 'get_%s_filename' % self.name, curry(cls._get_FIELD_filename, field=self))
487         setattr(cls, 'get_%s_url' % self.name, curry(cls._get_FIELD_url, field=self))
488         setattr(cls, 'get_%s_size' % self.name, curry(cls._get_FIELD_size, field=self))
489-        setattr(cls, 'save_%s_file' % self.name, lambda instance, filename, raw_contents, save=True: instance._save_FIELD_file(self, filename, raw_contents, save))
490+        setattr(cls, 'save_%s_file' % self.name, lambda instance, filename, raw_field, save=True: instance._save_FIELD_file(self, filename, raw_field, save))
491         dispatcher.connect(self.delete_file, signal=signals.post_delete, sender=cls)
492 
493     def delete_file(self, instance):
494@@ -659,9 +659,9 @@
495         if new_data.get(upload_field_name, False):
496             func = getattr(new_object, 'save_%s_file' % self.name)
497             if rel:
498-                func(new_data[upload_field_name][0]["filename"], new_data[upload_field_name][0]["content"], save)
499+                func(new_data[upload_field_name][0]["filename"], new_data[upload_field_name][0], save)
500             else:
501-                func(new_data[upload_field_name]["filename"], new_data[upload_field_name]["content"], save)
502+                func(new_data[upload_field_name]["filename"], new_data[upload_field_name], save)
503 
504     def get_directory_name(self):
505         return os.path.normpath(datetime.datetime.now().strftime(self.upload_to))
506Index: django/conf/global_settings.py
507===================================================================
508--- django/conf/global_settings.py      (revision 5100)
509+++ django/conf/global_settings.py      (working copy)
510@@ -240,6 +240,20 @@
511 # isExistingURL validator.
512 URL_VALIDATOR_USER_AGENT = "Django/0.96pre (http://www.djangoproject.com)"
513 
514+# The directory to place streamed file uploads. The web server needs write
515+# permissions on this directory.
516+# If this is None, streaming uploads are disabled.
517+FILE_UPLOAD_DIR = None
518+
519+
520+# The minimum size of a POST before file uploads are streamed to disk.
521+# Any less than this number, and the file is uploaded to memory.
522+# Size is in bytes.
523+STREAMING_MIN_POST_SIZE = 512 * (2**10)
524+
525+
526+
527+
528 ##############
529 # MIDDLEWARE #
530 ##############
531@@ -335,3 +349,5 @@
532 
533 # The list of directories to search for fixtures
534 FIXTURE_DIRS = ()
535+
536+
537Index: django/core/handlers/wsgi.py
538===================================================================
539--- django/core/handlers/wsgi.py        (revision 5100)
540+++ django/core/handlers/wsgi.py        (working copy)
541@@ -111,7 +111,14 @@
542             if self.environ.get('CONTENT_TYPE', '').startswith('multipart'):
543                 header_dict = dict([(k, v) for k, v in self.environ.items() if k.startswith('HTTP_')])
544                 header_dict['Content-Type'] = self.environ.get('CONTENT_TYPE', '')
545-                self._post, self._files = http.parse_file_upload(header_dict, self.raw_post_data)
546+                header_dict['Content-Length'] = self.environ.get('CONTENT_LENGTH', '')
547+                header_dict['X-Progress-ID'] = self.environ.get('HTTP_X_PROGRESS_ID', '')
548+                try:
549+                    self._post, self._files = http.parse_file_upload(header_dict, self.environ['wsgi.input'], self.META['UPLOAD_PROGRESS_ID'])
550+                except:
551+                    self._post, self._files = {}, {} # make sure we dont read the input stream again
552+                    raise
553+                self._raw_post_data = None # raw data is not available for streamed multipart messages
554             else:
555                 self._post, self._files = http.QueryDict(self.raw_post_data), datastructures.MultiValueDict()
556         else:
557@@ -167,6 +174,17 @@
558             buf.close()
559             return self._raw_post_data
560 
561+    def _get_file_progress_id(self):
562+        """
563+        Returns the Progress ID of the request,
564+        usually provided if there is a file upload
565+        going on.
566+        Returns ``None`` if no progress ID is specified.
567+        """
568+        return self._get_file_progress_from_args(self.environ,
569+                                                 self.GET,
570+                                                 self._req.args)
571+
572     GET = property(_get_get, _set_get)
573     POST = property(_get_post, _set_post)
574     COOKIES = property(_get_cookies, _set_cookies)
575Index: django/core/handlers/base.py
576===================================================================
577--- django/core/handlers/base.py        (revision 5100)
578+++ django/core/handlers/base.py        (working copy)
579@@ -5,7 +5,7 @@
580 
581 class BaseHandler(object):
582     def __init__(self):
583-        self._request_middleware = self._view_middleware = self._response_middleware = self._exception_middleware = None
584+        self._upload_middleware = self._request_middleware = self._view_middleware = self._response_middleware = self._exception_middleware = None
585 
586     def load_middleware(self):
587         """
588@@ -19,6 +19,7 @@
589         self._view_middleware = []
590         self._response_middleware = []
591         self._exception_middleware = []
592+        self._upload_middleware = []
593         for middleware_path in settings.MIDDLEWARE_CLASSES:
594             try:
595                 dot = middleware_path.rindex('.')
596@@ -47,6 +48,8 @@
597                 self._response_middleware.insert(0, mw_instance.process_response)
598             if hasattr(mw_instance, 'process_exception'):
599                 self._exception_middleware.insert(0, mw_instance.process_exception)
600+            if hasattr(mw_instance, 'process_upload'):
601+                self._upload_middleware.append(mw_instance.process_upload)
602 
603     def get_response(self, request):
604         "Returns an HttpResponse object for the given HttpRequest"
605Index: django/core/handlers/modpython.py
606===================================================================
607--- django/core/handlers/modpython.py   (revision 5100)
608+++ django/core/handlers/modpython.py   (working copy)
609@@ -47,7 +47,12 @@
610     def _load_post_and_files(self):
611         "Populates self._post and self._files"
612         if 'content-type' in self._req.headers_in and self._req.headers_in['content-type'].startswith('multipart'):
613-            self._post, self._files = http.parse_file_upload(self._req.headers_in, self.raw_post_data)
614+            self._raw_post_data = None # raw data is not available for streamed multipart messages
615+            try:
616+                self._post, self._files = http.parse_file_upload(self._req.headers_in, self._req, self.META['UPLOAD_PROGRESS_ID'])
617+            except:
618+                self._post, self._files = {}, {} # make sure we dont read the input stream again
619+                raise
620         else:
621             self._post, self._files = http.QueryDict(self.raw_post_data), datastructures.MultiValueDict()
622 
623@@ -92,20 +97,21 @@
624                 'AUTH_TYPE':         self._req.ap_auth_type,
625                 'CONTENT_LENGTH':    self._req.clength, # This may be wrong
626                 'CONTENT_TYPE':      self._req.content_type, # This may be wrong
627-                'GATEWAY_INTERFACE': 'CGI/1.1',
628-                'PATH_INFO':         self._req.path_info,
629-                'PATH_TRANSLATED':   None, # Not supported
630-                'QUERY_STRING':      self._req.args,
631-                'REMOTE_ADDR':       self._req.connection.remote_ip,
632-                'REMOTE_HOST':       None, # DNS lookups not supported
633-                'REMOTE_IDENT':      self._req.connection.remote_logname,
634-                'REMOTE_USER':       self._req.user,
635-                'REQUEST_METHOD':    self._req.method,
636-                'SCRIPT_NAME':       None, # Not supported
637-                'SERVER_NAME':       self._req.server.server_hostname,
638-                'SERVER_PORT':       self._req.server.port,
639-                'SERVER_PROTOCOL':   self._req.protocol,
640-                'SERVER_SOFTWARE':   'mod_python'
641+                'GATEWAY_INTERFACE':  'CGI/1.1',
642+                'PATH_INFO':          self._req.path_info,
643+                'PATH_TRANSLATED':    None, # Not supported
644+                'QUERY_STRING':       self._req.args,
645+                'REMOTE_ADDR':        self._req.connection.remote_ip,
646+                'REMOTE_HOST':        None, # DNS lookups not supported
647+                'REMOTE_IDENT':       self._req.connection.remote_logname,
648+                'REMOTE_USER':        self._req.user,
649+                'REQUEST_METHOD':     self._req.method,
650+                'SCRIPT_NAME':        None, # Not supported
651+                'SERVER_NAME':        self._req.server.server_hostname,
652+                'SERVER_PORT':        self._req.server.port,
653+                'SERVER_PROTOCOL':    self._req.protocol,
654+                'UPLOAD_PROGRESS_ID': self._get_file_progress_id(),
655+                'SERVER_SOFTWARE':    'mod_python'
656             }
657             for key, value in self._req.headers_in.items():
658                 key = 'HTTP_' + key.upper().replace('-', '_')
659@@ -122,6 +128,18 @@
660     def _get_method(self):
661         return self.META['REQUEST_METHOD'].upper()
662 
663+    def _get_file_progress_id(self):
664+        """
665+        Returns the Progress ID of the request,
666+        usually provided if there is a file upload
667+        going on.
668+        Returns ``None`` if no progress ID is specified.
669+        """
670+        return self._get_file_progress_from_args(self._req.headers_in,
671+                                                 self.GET,
672+                                                 self._req.args)
673+
674+
675     GET = property(_get_get, _set_get)
676     POST = property(_get_post, _set_post)
677     COOKIES = property(_get_cookies, _set_cookies)
678Index: tests/modeltests/test_client/views.py
679===================================================================
680--- tests/modeltests/test_client/views.py       (revision 5100)
681+++ tests/modeltests/test_client/views.py       (working copy)
682@@ -44,6 +44,12 @@
683 
684     return HttpResponse(t.render(c))
685 
686+def post_file_view(request):
687+    "A view that expects a multipart post and returns a file in the context"
688+    t = Template('File {{ file.filename }} received', name='POST Template')
689+    c = Context({'file': request.FILES['file_file']})
690+    return HttpResponse(t.render(c))
691+
692 def redirect_view(request):
693     "A view that redirects all requests to the GET view"
694     return HttpResponseRedirect('/test_client/get_view/')
695Index: tests/modeltests/test_client/models.py
696===================================================================
697--- tests/modeltests/test_client/models.py      (revision 5100)
698+++ tests/modeltests/test_client/models.py      (working copy)
699@@ -75,6 +75,21 @@
700         self.assertEqual(response.template.name, "Book template")
701         self.assertEqual(response.content, "Blink - Malcolm Gladwell")
702 
703+    def test_post_file_view(self):
704+        "POST this python file to a view"
705+        import os, tempfile
706+        from django.conf import settings
707+        file = __file__.replace('.pyc', '.py')
708+        for upload_dir in [None, tempfile.gettempdir()]:
709+            settings.FILE_UPLOAD_DIR = upload_dir
710+            post_data = { 'name': file, 'file': open(file) }
711+            response = self.client.post('/test_client/post_file_view/', post_data)
712+            self.failUnless('models.py' in response.context['file']['filename'])
713+            self.failUnless(len(response.context['file']['content']) == os.path.getsize(file))
714+            if upload_dir:
715+                self.failUnless(response.context['file']['tmpfilename'])
716+
717+
718     def test_redirect(self):
719         "GET a URL that redirects elsewhere"
720         response = self.client.get('/test_client/redirect_view/')
721Index: tests/modeltests/test_client/urls.py
722===================================================================
723--- tests/modeltests/test_client/urls.py        (revision 5100)
724+++ tests/modeltests/test_client/urls.py        (working copy)
725@@ -4,6 +4,7 @@
726 urlpatterns = patterns('',
727     (r'^get_view/$', views.get_view),
728     (r'^post_view/$', views.post_view),
729+    (r'^post_file_view/$', views.post_file_view),
730     (r'^raw_post_view/$', views.raw_post_view),
731     (r'^redirect_view/$', views.redirect_view),
732     (r'^form_view/$', views.form_view),
733Index: docs/request_response.txt
734===================================================================
735--- docs/request_response.txt   (revision 5100)
736+++ docs/request_response.txt   (working copy)
737@@ -72,13 +72,25 @@
738 ``FILES``
739     A dictionary-like object containing all uploaded files. Each key in
740     ``FILES`` is the ``name`` from the ``<input type="file" name="" />``. Each
741-    value in ``FILES`` is a standard Python dictionary with the following three
742+    value in ``FILES`` is a standard Python dictionary with the following four
743     keys:
744 
745         * ``filename`` -- The name of the uploaded file, as a Python string.
746         * ``content-type`` -- The content type of the uploaded file.
747         * ``content`` -- The raw content of the uploaded file.
748+        * ``content-length`` -- The length of the content in bytes.
749 
750+    If streaming file uploads are enabled two additional keys
751+    describing the uploaded file will be present:
752+
753+       * ``tmpfilename`` -- The filename for the temporary file.
754+       * ``tmpfile`` -- An open file object for the temporary file.
755+
756+    The temporary file will be removed when the request finishes.
757+
758+    Note that accessing ``content`` when streaming uploads are enabled
759+    will read the whole file into memory which may not be what you want.
760+
761     Note that ``FILES`` will only contain data if the request method was POST
762     and the ``<form>`` that posted to the request had
763     ``enctype="multipart/form-data"``. Otherwise, ``FILES`` will be a blank
764Index: docs/settings.txt
765===================================================================
766--- docs/settings.txt   (revision 5100)
767+++ docs/settings.txt   (working copy)
768@@ -437,6 +437,15 @@
769 
770 .. _Testing Django Applications: ../testing/
771 
772+FILE_UPLOAD_DIR
773+---------------
774+
775+Default: ``None``
776+
777+Path to a directory where temporary files should be written during
778+file uploads. Leaving this as ``None`` will disable streaming file uploads,
779+and cause all uploaded files to be stored (temporarily) in memory.
780+
781 IGNORABLE_404_ENDS
782 ------------------
783 
784@@ -780,6 +789,16 @@
785 
786 .. _site framework docs: ../sites/
787 
788+STREAMING_MIN_POST_SIZE
789+-----------------------
790+
791+Default: 524288 (``512*1024``)
792+
793+An integer specifying the minimum number of bytes that has to be
794+received (in a POST) for file upload streaming to take place. Any
795+request smaller than this will be handled in memory.
796+Note: ``FILE_UPLOAD_DIR`` has to be defined to enable streaming.
797+
798 TEMPLATE_CONTEXT_PROCESSORS
799 ---------------------------
800 
801Index: docs/forms.txt
802===================================================================
803--- docs/forms.txt      (revision 5100)
804+++ docs/forms.txt      (working copy)
805@@ -475,6 +475,19 @@
806    new_data = request.POST.copy()
807    new_data.update(request.FILES)
808 
809+Streaming file uploads.
810+-----------------------
811+
812+File uploads will be read into memory by default. This works fine for
813+small to medium sized uploads (from 1MB to 100MB depending on your
814+setup and usage). If you want to support larger uploads you can enable
815+upload streaming where only a small part of the file will be in memory
816+at any time. To do this you need to specify the ``FILE_UPLOAD_DIR``
817+setting (see the settings_ document for more details).
818+
819+See `request object`_ for more details about ``request.FILES`` objects
820+with streaming file uploads enabled.
821+
822 Validators
823 ==========
824 
825@@ -693,3 +706,4 @@
826 .. _`generic views`: ../generic_views/
827 .. _`models API`: ../model-api/
828 .. _settings: ../settings/
829+.. _request object: ../request_response/#httprequest-objects