Ticket #15785: limited_stream_lazy_plus_extra_tests.diff

File limited_stream_lazy_plus_extra_tests.diff, 17.3 KB (added by Tom Christie, 14 years ago)

  • tests/regressiontests/file_uploads/tests.py

     
    88
    99from django.core.files import temp as tempfile
    1010from django.core.files.uploadedfile import SimpleUploadedFile
    11 from django.http.multipartparser import MultiPartParser
     11from django.http.multipartparser import MultiPartParser, MultiPartParserError
    1212from django.test import TestCase, client
    1313from django.utils import simplejson
    1414from django.utils import unittest
    … …  
    151151        got = simplejson.loads(self.client.request(**r).content)
    152152        self.assertTrue(len(got['file']) < 256, "Got a long file name (%s characters)." % len(got['file']))
    153153
     154    def test_truncated_multipart_handled_gracefully(self):
     155        """
     156        If passed an incomplete multipart message, MultiPartParser does not
     157        attempt to read beyond the end of the stream, and simply will handle
     158        the part that can be parsed gracefully.
     159        """
     160        payload = "\r\n".join([
     161            '--' + client.BOUNDARY,
     162            'Content-Disposition: form-data; name="file"; filename="foo.txt"',
     163            'Content-Type: application/octet-stream',
     164            '',
     165            'file contents'
     166            '--' + client.BOUNDARY + '--',
     167            '',
     168        ])
     169        payload = payload[:-10]
     170        r = {
     171            'CONTENT_LENGTH': len(payload),
     172            'CONTENT_TYPE':   client.MULTIPART_CONTENT,
     173            'PATH_INFO':      "/file_uploads/echo/",
     174            'REQUEST_METHOD': 'POST',
     175            'wsgi.input':     client.FakePayload(payload),
     176        }
     177        got = simplejson.loads(self.client.request(**r).content)
     178        self.assertEquals(got, {})
     179
     180    def test_empty_multipart_raises_error(self):
     181        """
     182        If passed an empty multipart message, MultiPartParser will throw
     183        MultiPartParserError.
     184        """
     185        r = {
     186            'CONTENT_LENGTH': 0,
     187            'CONTENT_TYPE':   client.MULTIPART_CONTENT,
     188            'PATH_INFO':      "/file_uploads/echo/",
     189            'REQUEST_METHOD': 'POST',
     190            'wsgi.input':     client.FakePayload(''),
     191        }
     192        self.assertRaises(MultiPartParserError, lambda r: self.client.request(**r), r)
     193
    154194    def test_custom_upload_handler(self):
    155195        # A small file (under the 5M quota)
    156196        smallfile = tempfile.NamedTemporaryFile()

  • tests/regressiontests/test_client_regress/views.py

     
    9696    "A view that is requested with GET and accesses request.raw_post_data. Refs #14753."
    9797    return HttpResponse(request.raw_post_data)
    9898
     99def read_all(request):
     100    "A view that is requested with accesses request.read()."
     101    return HttpResponse(request.read())
     102
     103def read_buffer(request):
     104    "A view that is requested with accesses request.read(LARGE_BUFFER)."
     105    return HttpResponse(request.read(99999))
     106
    99107def request_context_view(request):
    100108    # Special attribute that won't be present on a plain HttpRequest
    101109    request.special_path = request.path

  • tests/regressiontests/test_client_regress/models.py

     
    900900        response = self.client.get("/test_client_regress/request_methods/")
    901901        self.assertEqual(response.template, None)
    902902
    903 class RawPostDataTest(TestCase):
    904     "Access to request.raw_post_data from the test client."
    905     def test_raw_post_data(self):
    906         # Refs #14753
    907         try:
    908             response = self.client.get("/test_client_regress/raw_post_data/")
    909         except AssertionError:
    910             self.fail("Accessing request.raw_post_data from a view fetched with GET by the test client shouldn't fail.")
     903
     904class ReadLimitedStreamTest(TestCase):
     905    """
     906    Tests that ensure that HttpRequest.raw_post_data, HttpRequest.read() and
     907    HttpRequest.read(BUFFER) have proper LimitedStream behaviour.
     908
     909    Refs #14753, #15785
     910    """
     911    def test_raw_post_data_from_empty_request(self):
     912        """HttpRequest.raw_post_data on a test client GET request should return
     913        the empty string."""
     914        self.assertEquals(self.client.get("/test_client_regress/raw_post_data/").content, '')
     915
     916    def test_read_from_empty_request(self):
     917        """HttpRequest.read() on a test client GET request should return the
     918        empty string."""
     919        self.assertEquals(self.client.get("/test_client_regress/read_all/").content, '')
     920
     921    def test_read_numbytes_from_empty_request(self):
     922        """HttpRequest.read(LARGE_BUFFER) on a test client GET request should
     923        return the empty string."""
     924        self.assertEquals(self.client.get("/test_client_regress/read_buffer/").content, '')
     925
     926    def test_read_from_nonempty_request(self):
     927        """HttpRequest.read() on a test client PUT request with some payload
     928        should return that payload."""
     929        payload = 'foobar'
     930        self.assertEquals(self.client.put("/test_client_regress/read_all/",
     931                                          data=payload,
     932                                          content_type='text/plain').content, payload)
     933
     934    def test_read_numbytes_from_nonempty_request(self):
     935        """HttpRequest.read(LARGE_BUFFER) on a test client PUT request with
     936        some payload should return that payload."""
     937        payload = 'foobar'
     938        self.assertEquals(self.client.put("/test_client_regress/read_buffer/",
     939                                          data=payload,
     940                                          content_type='text/plain').content, payload)

  • tests/regressiontests/test_client_regress/urls.py

     
    2727    (r'^check_headers/$', views.check_headers),
    2828    (r'^check_headers_redirect/$', RedirectView.as_view(url='/test_client_regress/check_headers/')),
    2929    (r'^raw_post_data/$', views.raw_post_data),
     30    (r'^read_all/$', views.read_all),
     31    (r'^read_buffer/$', views.read_buffer),
    3032    (r'^request_context_view/$', views.request_context_view),
    3133)

  • tests/regressiontests/requests/tests.py

     
    156156        self.assertEqual(stream.read(), '')
    157157
    158158    def test_stream(self):
    159         request = WSGIRequest({'REQUEST_METHOD': 'POST', 'wsgi.input': StringIO('name=value')})
     159        payload = 'name=value'
     160        request = WSGIRequest({'REQUEST_METHOD': 'POST',
     161                               'CONTENT_LENGTH': len(payload),
     162                               'wsgi.input': StringIO(payload)})
    160163        self.assertEqual(request.read(), 'name=value')
    161164
    162165    def test_read_after_value(self):
    … …  
    164167        Reading from request is allowed after accessing request contents as
    165168        POST or raw_post_data.
    166169        """
    167         request = WSGIRequest({'REQUEST_METHOD': 'POST', 'wsgi.input': StringIO('name=value')})
     170        payload = 'name=value'
     171        request = WSGIRequest({'REQUEST_METHOD': 'POST',
     172                               'CONTENT_LENGTH': len(payload),
     173                               'wsgi.input': StringIO(payload)})
    168174        self.assertEqual(request.POST, {u'name': [u'value']})
    169175        self.assertEqual(request.raw_post_data, 'name=value')
    170176        self.assertEqual(request.read(), 'name=value')
    … …  
    174180        Construction of POST or raw_post_data is not allowed after reading
    175181        from request.
    176182        """
    177         request = WSGIRequest({'REQUEST_METHOD': 'POST', 'wsgi.input': StringIO('name=value')})
     183        payload = 'name=value'
     184        request = WSGIRequest({'REQUEST_METHOD': 'POST',
     185                               'CONTENT_LENGTH': len(payload),
     186                               'wsgi.input': StringIO(payload)})
    178187        self.assertEqual(request.read(2), 'na')
    179188        self.assertRaises(Exception, lambda: request.raw_post_data)
    180189        self.assertEqual(request.POST, {})
    … …  
    201210        self.assertRaises(Exception, lambda: request.raw_post_data)
    202211
    203212    def test_read_by_lines(self):
    204         request = WSGIRequest({'REQUEST_METHOD': 'POST', 'wsgi.input': StringIO('name=value')})
     213        payload = 'name=value'
     214        request = WSGIRequest({'REQUEST_METHOD': 'POST',
     215                               'CONTENT_LENGTH': len(payload),
     216                               'wsgi.input': StringIO(payload)})
    205217        self.assertEqual(list(request), ['name=value'])
    206218
    207219    def test_POST_after_raw_post_data_read(self):
    208220        """
    209221        POST should be populated even if raw_post_data is read first
    210222        """
    211         request = WSGIRequest({'REQUEST_METHOD': 'POST', 'wsgi.input': StringIO('name=value')})
     223        payload = 'name=value'
     224        request = WSGIRequest({'REQUEST_METHOD': 'POST',
     225                               'CONTENT_LENGTH': len(payload),
     226                               'wsgi.input': StringIO(payload)})
    212227        raw_data = request.raw_post_data
    213228        self.assertEqual(request.POST, {u'name': [u'value']})
    214229
    … …  
    217232        POST should be populated even if raw_post_data is read first, and then
    218233        the stream is read second.
    219234        """
    220         request = WSGIRequest({'REQUEST_METHOD': 'POST', 'wsgi.input': StringIO('name=value')})
     235        payload = 'name=value'
     236        request = WSGIRequest({'REQUEST_METHOD': 'POST',
     237                               'CONTENT_LENGTH': len(payload),
     238                               'wsgi.input': StringIO(payload)})
    221239        raw_data = request.raw_post_data
    222240        self.assertEqual(request.read(1), u'n')
    223241        self.assertEqual(request.POST, {u'name': [u'value']})

  • django/http/multipartparser.py

     
    3333    A rfc2388 multipart/form-data parser.
    3434
    3535    ``MultiValueDict.parse()`` reads the input stream in ``chunk_size`` chunks
    36     and returns a tuple of ``(MultiValueDict(POST), MultiValueDict(FILES))``. If
     36    and returns a tuple of ``(MultiValueDict(POST), MultiValueDict(FILES))``.
    3737    """
    3838    def __init__(self, META, input_data, upload_handlers, encoding=None):
    3939        """
    … …  
    6565            raise MultiPartParserError('Invalid boundary in multipart: %s' % boundary)
    6666
    6767
    68         #
    6968        # Content-Length should contain the length of the body we are about
    7069        # to receive.
    71         #
    7270        try:
    7371            content_length = int(META.get('HTTP_CONTENT_LENGTH', META.get('CONTENT_LENGTH',0)))
    7472        except (ValueError, TypeError):
    75             # For now set it to 0; we'll try again later on down.
    7673            content_length = 0
    7774
    7875        if content_length <= 0:
    … …  
    105102        encoding = self._encoding
    106103        handlers = self._upload_handlers
    107104
    108         limited_input_data = LimitBytes(self._input_data, self._content_length)
    109 
    110105        # See if the handler will want to take care of the parsing.
    111106        # This allows overriding everything if somebody wants it.
    112107        for handler in handlers:
    113             result = handler.handle_raw_input(limited_input_data,
     108            result = handler.handle_raw_input(self._input_data,
    114109                                              self._meta,
    115110                                              self._content_length,
    116111                                              self._boundary,
    … …  
    123118        self._files = MultiValueDict()
    124119
    125120        # Instantiate the parser and stream:
    126         stream = LazyStream(ChunkIter(limited_input_data, self._chunk_size))
     121        stream = LazyStream(ChunkIter(self._input_data, self._chunk_size))
    127122
    128123        # Whether or not to signal a file-completion at the beginning of the loop.
    129124        old_field_name = None
    … …  
    218213                    exhaust(stream)
    219214        except StopUpload, e:
    220215            if not e.connection_reset:
    221                 exhaust(limited_input_data)
     216                exhaust(self._input_data)
    222217        else:
    223218            # Make sure that the request data is all fed
    224             exhaust(limited_input_data)
     219            exhaust(self._input_data)
    225220
    226221        # Signal that the upload has completed.
    227222        for handler in handlers:
    … …  
    383378    def __iter__(self):
    384379        return self
    385380
    386 class LimitBytes(object):
    387     """ Limit bytes for a file object. """
    388     def __init__(self, fileobject, length):
    389         self._file = fileobject
    390         self.remaining = length
    391 
    392     def read(self, num_bytes=None):
    393         """
    394         Read data from the underlying file.
    395         If you ask for too much or there isn't anything left,
    396         this will raise an InputStreamExhausted error.
    397         """
    398         if self.remaining <= 0:
    399             raise InputStreamExhausted()
    400         if num_bytes is None:
    401             num_bytes = self.remaining
    402         else:
    403             num_bytes = min(num_bytes, self.remaining)
    404         self.remaining -= num_bytes
    405         return self._file.read(num_bytes)
    406 
    407381class InterBoundaryIter(object):
    408382    """
    409383    A Producer that will iterate over boundaries.

  • django/http/__init__.py

     
    237237        if not hasattr(self, '_raw_post_data'):
    238238            if self._read_started:
    239239                raise Exception("You cannot access raw_post_data after reading from request's data stream")
    240             try:
    241                 content_length = int(self.META.get('CONTENT_LENGTH', 0))
    242             except (ValueError, TypeError):
    243                 # If CONTENT_LENGTH was empty string or not an integer, don't
    244                 # error out. We've also seen None passed in here (against all
    245                 # specs, but see ticket #8259), so we handle TypeError as well.
    246                 content_length = 0
    247             if content_length:
    248                 self._raw_post_data = self.read(content_length)
    249             else:
    250                 self._raw_post_data = self.read()
     240            self._raw_post_data = self.read()
    251241            self._stream = StringIO(self._raw_post_data)
    252242        return self._raw_post_data
    253243    raw_post_data = property(_get_raw_post_data)

  • django/core/handlers/wsgi.py

     
    135135        self.META['SCRIPT_NAME'] = script_name
    136136        self.method = environ['REQUEST_METHOD'].upper()
    137137        self._post_parse_error = False
    138         if type(socket._fileobject) is type and isinstance(self.environ['wsgi.input'], socket._fileobject):
    139             # Under development server 'wsgi.input' is an instance of
    140             # socket._fileobject which hangs indefinitely on reading bytes past
    141             # available count. To prevent this it's wrapped in LimitedStream
    142             # that doesn't read past Content-Length bytes.
    143             #
    144             # This is not done for other kinds of inputs (like flup's FastCGI
    145             # streams) beacuse they don't suffer from this problem and we can
    146             # avoid using another wrapper with its own .read and .readline
    147             # implementation.
    148             #
    149             # The type check is done because for some reason, AppEngine
    150             # implements _fileobject as a function, not a class.
    151             try:
    152                 content_length = int(self.environ.get('CONTENT_LENGTH', 0))
    153             except (ValueError, TypeError):
    154                 content_length = 0
    155             self._stream = LimitedStream(self.environ['wsgi.input'], content_length)
    156         else:
    157             self._stream = self.environ['wsgi.input']
    158138        self._read_started = False
    159139
    160140    def __repr__(self):
    … …  
    191171        return 'wsgi.url_scheme' in self.environ \
    192172            and self.environ['wsgi.url_scheme'] == 'https'
    193173
     174    def _get_stream(self):
     175        if not hasattr(self, '_limited_stream'):
     176            try:
     177                content_length = int(self.environ.get('CONTENT_LENGTH', 0))
     178            except (ValueError, TypeError):
     179                content_length = 0
     180
     181            self._limited_stream = LimitedStream(self.environ['wsgi.input'], content_length)
     182
     183        return self._limited_stream
     184
     185    def _set_stream(self, stream):
     186        self._limited_stream = stream
     187
    194188    def _get_request(self):
    195189        if not hasattr(self, '_request'):
    196190            self._request = datastructures.MergeDict(self.POST, self.GET)
    … …  
    226220            self._load_post_and_files()
    227221        return self._files
    228222
     223    _stream = property(_get_stream, _set_stream)
    229224    GET = property(_get_get, _set_get)
    230225    POST = property(_get_post, _set_post)
    231226    COOKIES = property(_get_cookies, _set_cookies)
Back to Top