Code

Ticket #15785: limited_stream_lazy_plus_extra_tests.4.diff

File limited_stream_lazy_plus_extra_tests.4.diff, 16.5 KB (added by tomchristie, 3 years ago)
Line 
1Index: tests/regressiontests/file_uploads/tests.py
2===================================================================
3--- tests/regressiontests/file_uploads/tests.py (revision 16013)
4+++ tests/regressiontests/file_uploads/tests.py (working copy)
5@@ -8,7 +8,7 @@
6 
7 from django.core.files import temp as tempfile
8 from django.core.files.uploadedfile import SimpleUploadedFile
9-from django.http.multipartparser import MultiPartParser
10+from django.http.multipartparser import MultiPartParser, MultiPartParserError
11 from django.test import TestCase, client
12 from django.utils import simplejson
13 from django.utils import unittest
14@@ -151,6 +151,47 @@
15         got = simplejson.loads(self.client.request(**r).content)
16         self.assertTrue(len(got['file']) < 256, "Got a long file name (%s characters)." % len(got['file']))
17 
18+    def test_truncated_multipart_handled_gracefully(self):
19+        """
20+        If passed an incomplete multipart message, MultiPartParser does not
21+        attempt to read beyond the end of the stream, and simply will handle
22+        the part that can be parsed gracefully.
23+        """
24+        payload = "\r\n".join([
25+            '--' + client.BOUNDARY,
26+            'Content-Disposition: form-data; name="file"; filename="foo.txt"',
27+            'Content-Type: application/octet-stream',
28+            '',
29+            'file contents'
30+            '--' + client.BOUNDARY + '--',
31+            '',
32+        ])
33+        payload = payload[:-10]
34+        r = {
35+            'CONTENT_LENGTH': len(payload),
36+            'CONTENT_TYPE': client.MULTIPART_CONTENT,
37+            'PATH_INFO': '/file_uploads/echo/',
38+            'REQUEST_METHOD': 'POST',
39+            'wsgi.input': client.FakePayload(payload),
40+        }
41+        got = simplejson.loads(self.client.request(**r).content)
42+        self.assertEquals(got, {})
43+
44+    def test_empty_multipart_handled_gracefully(self):
45+        """
46+        If passed an empty multipart message, MultiPartParser will return
47+        an empty QueryDict.
48+        """
49+        r = {
50+            'CONTENT_LENGTH': 0,
51+            'CONTENT_TYPE': client.MULTIPART_CONTENT,
52+            'PATH_INFO': '/file_uploads/echo/',
53+            'REQUEST_METHOD': 'POST',
54+            'wsgi.input': client.FakePayload(''),
55+        }
56+        got = simplejson.loads(self.client.request(**r).content)
57+        self.assertEquals(got, {})
58+
59     def test_custom_upload_handler(self):
60         # A small file (under the 5M quota)
61         smallfile = tempfile.NamedTemporaryFile()
62Index: tests/regressiontests/test_client_regress/views.py
63===================================================================
64--- tests/regressiontests/test_client_regress/views.py  (revision 16013)
65+++ tests/regressiontests/test_client_regress/views.py  (working copy)
66@@ -96,6 +96,14 @@
67     "A view that is requested with GET and accesses request.raw_post_data. Refs #14753."
68     return HttpResponse(request.raw_post_data)
69 
70+def read_all(request):
71+    "A view that is requested with accesses request.read()."
72+    return HttpResponse(request.read())
73+
74+def read_buffer(request):
75+    "A view that is requested with accesses request.read(LARGE_BUFFER)."
76+    return HttpResponse(request.read(99999))
77+
78 def request_context_view(request):
79     # Special attribute that won't be present on a plain HttpRequest
80     request.special_path = request.path
81Index: tests/regressiontests/test_client_regress/models.py
82===================================================================
83--- tests/regressiontests/test_client_regress/models.py (revision 16013)
84+++ tests/regressiontests/test_client_regress/models.py (working copy)
85@@ -900,11 +900,41 @@
86         response = self.client.get("/test_client_regress/request_methods/")
87         self.assertEqual(response.template, None)
88 
89-class RawPostDataTest(TestCase):
90-    "Access to request.raw_post_data from the test client."
91-    def test_raw_post_data(self):
92-        # Refs #14753
93-        try:
94-            response = self.client.get("/test_client_regress/raw_post_data/")
95-        except AssertionError:
96-            self.fail("Accessing request.raw_post_data from a view fetched with GET by the test client shouldn't fail.")
97+
98+class ReadLimitedStreamTest(TestCase):
99+    """
100+    Tests that ensure that HttpRequest.raw_post_data, HttpRequest.read() and
101+    HttpRequest.read(BUFFER) have proper LimitedStream behaviour.
102+
103+    Refs #14753, #15785
104+    """
105+    def test_raw_post_data_from_empty_request(self):
106+        """HttpRequest.raw_post_data on a test client GET request should return
107+        the empty string."""
108+        self.assertEquals(self.client.get("/test_client_regress/raw_post_data/").content, '')
109+
110+    def test_read_from_empty_request(self):
111+        """HttpRequest.read() on a test client GET request should return the
112+        empty string."""
113+        self.assertEquals(self.client.get("/test_client_regress/read_all/").content, '')
114+
115+    def test_read_numbytes_from_empty_request(self):
116+        """HttpRequest.read(LARGE_BUFFER) on a test client GET request should
117+        return the empty string."""
118+        self.assertEquals(self.client.get("/test_client_regress/read_buffer/").content, '')
119+
120+    def test_read_from_nonempty_request(self):
121+        """HttpRequest.read() on a test client PUT request with some payload
122+        should return that payload."""
123+        payload = 'foobar'
124+        self.assertEquals(self.client.put("/test_client_regress/read_all/",
125+                                          data=payload,
126+                                          content_type='text/plain').content, payload)
127+
128+    def test_read_numbytes_from_nonempty_request(self):
129+        """HttpRequest.read(LARGE_BUFFER) on a test client PUT request with
130+        some payload should return that payload."""
131+        payload = 'foobar'
132+        self.assertEquals(self.client.put("/test_client_regress/read_buffer/",
133+                                          data=payload,
134+                                          content_type='text/plain').content, payload)
135Index: tests/regressiontests/test_client_regress/urls.py
136===================================================================
137--- tests/regressiontests/test_client_regress/urls.py   (revision 16013)
138+++ tests/regressiontests/test_client_regress/urls.py   (working copy)
139@@ -27,5 +27,7 @@
140     (r'^check_headers/$', views.check_headers),
141     (r'^check_headers_redirect/$', RedirectView.as_view(url='/test_client_regress/check_headers/')),
142     (r'^raw_post_data/$', views.raw_post_data),
143+    (r'^read_all/$', views.read_all),
144+    (r'^read_buffer/$', views.read_buffer),
145     (r'^request_context_view/$', views.request_context_view),
146 )
147Index: tests/regressiontests/requests/tests.py
148===================================================================
149--- tests/regressiontests/requests/tests.py     (revision 16013)
150+++ tests/regressiontests/requests/tests.py     (working copy)
151@@ -156,7 +156,10 @@
152         self.assertEqual(stream.read(), '')
153 
154     def test_stream(self):
155-        request = WSGIRequest({'REQUEST_METHOD': 'POST', 'wsgi.input': StringIO('name=value')})
156+        payload = 'name=value'
157+        request = WSGIRequest({'REQUEST_METHOD': 'POST',
158+                               'CONTENT_LENGTH': len(payload),
159+                               'wsgi.input': StringIO(payload)})
160         self.assertEqual(request.read(), 'name=value')
161 
162     def test_read_after_value(self):
163@@ -164,7 +167,10 @@
164         Reading from request is allowed after accessing request contents as
165         POST or raw_post_data.
166         """
167-        request = WSGIRequest({'REQUEST_METHOD': 'POST', 'wsgi.input': StringIO('name=value')})
168+        payload = 'name=value'
169+        request = WSGIRequest({'REQUEST_METHOD': 'POST',
170+                               'CONTENT_LENGTH': len(payload),
171+                               'wsgi.input': StringIO(payload)})
172         self.assertEqual(request.POST, {u'name': [u'value']})
173         self.assertEqual(request.raw_post_data, 'name=value')
174         self.assertEqual(request.read(), 'name=value')
175@@ -174,7 +180,10 @@
176         Construction of POST or raw_post_data is not allowed after reading
177         from request.
178         """
179-        request = WSGIRequest({'REQUEST_METHOD': 'POST', 'wsgi.input': StringIO('name=value')})
180+        payload = 'name=value'
181+        request = WSGIRequest({'REQUEST_METHOD': 'POST',
182+                               'CONTENT_LENGTH': len(payload),
183+                               'wsgi.input': StringIO(payload)})
184         self.assertEqual(request.read(2), 'na')
185         self.assertRaises(Exception, lambda: request.raw_post_data)
186         self.assertEqual(request.POST, {})
187@@ -201,14 +210,20 @@
188         self.assertRaises(Exception, lambda: request.raw_post_data)
189 
190     def test_read_by_lines(self):
191-        request = WSGIRequest({'REQUEST_METHOD': 'POST', 'wsgi.input': StringIO('name=value')})
192+        payload = 'name=value'
193+        request = WSGIRequest({'REQUEST_METHOD': 'POST',
194+                               'CONTENT_LENGTH': len(payload),
195+                               'wsgi.input': StringIO(payload)})
196         self.assertEqual(list(request), ['name=value'])
197 
198     def test_POST_after_raw_post_data_read(self):
199         """
200         POST should be populated even if raw_post_data is read first
201         """
202-        request = WSGIRequest({'REQUEST_METHOD': 'POST', 'wsgi.input': StringIO('name=value')})
203+        payload = 'name=value'
204+        request = WSGIRequest({'REQUEST_METHOD': 'POST',
205+                               'CONTENT_LENGTH': len(payload),
206+                               'wsgi.input': StringIO(payload)})
207         raw_data = request.raw_post_data
208         self.assertEqual(request.POST, {u'name': [u'value']})
209 
210@@ -217,7 +232,10 @@
211         POST should be populated even if raw_post_data is read first, and then
212         the stream is read second.
213         """
214-        request = WSGIRequest({'REQUEST_METHOD': 'POST', 'wsgi.input': StringIO('name=value')})
215+        payload = 'name=value'
216+        request = WSGIRequest({'REQUEST_METHOD': 'POST',
217+                               'CONTENT_LENGTH': len(payload),
218+                               'wsgi.input': StringIO(payload)})
219         raw_data = request.raw_post_data
220         self.assertEqual(request.read(1), u'n')
221         self.assertEqual(request.POST, {u'name': [u'value']})
222Index: django/http/multipartparser.py
223===================================================================
224--- django/http/multipartparser.py      (revision 16013)
225+++ django/http/multipartparser.py      (working copy)
226@@ -33,7 +33,7 @@
227     A rfc2388 multipart/form-data parser.
228 
229     ``MultiValueDict.parse()`` reads the input stream in ``chunk_size`` chunks
230-    and returns a tuple of ``(MultiValueDict(POST), MultiValueDict(FILES))``. If
231+    and returns a tuple of ``(MultiValueDict(POST), MultiValueDict(FILES))``.
232     """
233     def __init__(self, META, input_data, upload_handlers, encoding=None):
234         """
235@@ -65,14 +65,11 @@
236             raise MultiPartParserError('Invalid boundary in multipart: %s' % boundary)
237 
238 
239-        #
240         # Content-Length should contain the length of the body we are about
241         # to receive.
242-        #
243         try:
244             content_length = int(META.get('HTTP_CONTENT_LENGTH', META.get('CONTENT_LENGTH',0)))
245         except (ValueError, TypeError):
246-            # For now set it to 0; we'll try again later on down.
247             content_length = 0
248 
249         if content_length <= 0:
250@@ -105,12 +102,10 @@
251         encoding = self._encoding
252         handlers = self._upload_handlers
253 
254-        limited_input_data = LimitBytes(self._input_data, self._content_length)
255-
256         # See if the handler will want to take care of the parsing.
257         # This allows overriding everything if somebody wants it.
258         for handler in handlers:
259-            result = handler.handle_raw_input(limited_input_data,
260+            result = handler.handle_raw_input(self._input_data,
261                                               self._meta,
262                                               self._content_length,
263                                               self._boundary,
264@@ -123,7 +118,7 @@
265         self._files = MultiValueDict()
266 
267         # Instantiate the parser and stream:
268-        stream = LazyStream(ChunkIter(limited_input_data, self._chunk_size))
269+        stream = LazyStream(ChunkIter(self._input_data, self._chunk_size))
270 
271         # Whether or not to signal a file-completion at the beginning of the loop.
272         old_field_name = None
273@@ -218,10 +213,10 @@
274                     exhaust(stream)
275         except StopUpload, e:
276             if not e.connection_reset:
277-                exhaust(limited_input_data)
278+                exhaust(self._input_data)
279         else:
280             # Make sure that the request data is all fed
281-            exhaust(limited_input_data)
282+            exhaust(self._input_data)
283 
284         # Signal that the upload has completed.
285         for handler in handlers:
286@@ -383,27 +378,6 @@
287     def __iter__(self):
288         return self
289 
290-class LimitBytes(object):
291-    """ Limit bytes for a file object. """
292-    def __init__(self, fileobject, length):
293-        self._file = fileobject
294-        self.remaining = length
295-
296-    def read(self, num_bytes=None):
297-        """
298-        Read data from the underlying file.
299-        If you ask for too much or there isn't anything left,
300-        this will raise an InputStreamExhausted error.
301-        """
302-        if self.remaining <= 0:
303-            raise InputStreamExhausted()
304-        if num_bytes is None:
305-            num_bytes = self.remaining
306-        else:
307-            num_bytes = min(num_bytes, self.remaining)
308-        self.remaining -= num_bytes
309-        return self._file.read(num_bytes)
310-
311 class InterBoundaryIter(object):
312     """
313     A Producer that will iterate over boundaries.
314Index: django/http/__init__.py
315===================================================================
316--- django/http/__init__.py     (revision 16013)
317+++ django/http/__init__.py     (working copy)
318@@ -237,17 +237,7 @@
319         if not hasattr(self, '_raw_post_data'):
320             if self._read_started:
321                 raise Exception("You cannot access raw_post_data after reading from request's data stream")
322-            try:
323-                content_length = int(self.META.get('CONTENT_LENGTH', 0))
324-            except (ValueError, TypeError):
325-                # If CONTENT_LENGTH was empty string or not an integer, don't
326-                # error out. We've also seen None passed in here (against all
327-                # specs, but see ticket #8259), so we handle TypeError as well.
328-                content_length = 0
329-            if content_length:
330-                self._raw_post_data = self.read(content_length)
331-            else:
332-                self._raw_post_data = self.read()
333+            self._raw_post_data = self.read()
334             self._stream = StringIO(self._raw_post_data)
335         return self._raw_post_data
336     raw_post_data = property(_get_raw_post_data)
337Index: django/core/handlers/wsgi.py
338===================================================================
339--- django/core/handlers/wsgi.py        (revision 16013)
340+++ django/core/handlers/wsgi.py        (working copy)
341@@ -135,26 +135,11 @@
342         self.META['SCRIPT_NAME'] = script_name
343         self.method = environ['REQUEST_METHOD'].upper()
344         self._post_parse_error = False
345-        if type(socket._fileobject) is type and isinstance(self.environ['wsgi.input'], socket._fileobject):
346-            # Under development server 'wsgi.input' is an instance of
347-            # socket._fileobject which hangs indefinitely on reading bytes past
348-            # available count. To prevent this it's wrapped in LimitedStream
349-            # that doesn't read past Content-Length bytes.
350-            #
351-            # This is not done for other kinds of inputs (like flup's FastCGI
352-            # streams) beacuse they don't suffer from this problem and we can
353-            # avoid using another wrapper with its own .read and .readline
354-            # implementation.
355-            #
356-            # The type check is done because for some reason, AppEngine
357-            # implements _fileobject as a function, not a class.
358-            try:
359-                content_length = int(self.environ.get('CONTENT_LENGTH', 0))
360-            except (ValueError, TypeError):
361-                content_length = 0
362-            self._stream = LimitedStream(self.environ['wsgi.input'], content_length)
363-        else:
364-            self._stream = self.environ['wsgi.input']
365+        try:
366+            content_length = int(self.environ.get('CONTENT_LENGTH'))
367+        except (ValueError, TypeError):
368+            content_length = 0
369+        self._stream = LimitedStream(self.environ['wsgi.input'], content_length)
370         self._read_started = False
371 
372     def __repr__(self):