Ticket #1484: 3013-streaming-file.diff

File 3013-streaming-file.diff, 7.4 KB (added by [530], 9 years ago)

Updated for new trunk

  • django/http/__init__.py

     
    22from pprint import pformat
    33from urllib import urlencode
    44from django.utils.datastructures import MultiValueDict
     5import cgi
     6from StringIO import StringIO
    57
    68try:
    79    # The mod_python version is more efficient, so try importing it first.
     
    3537    def get_full_path(self):
    3638        return ''
    3739
    38 def parse_file_upload(header_dict, post_data):
     40class FileDict(dict):
     41    "Keeps uploaded file as a file-like object and reads its content on demand"
     42    def __getitem__(self, name):
     43        if name=='content' and not 'content' in self:
     44            self['file'].seek(0, 2)
     45            size = self['file'].tell()
     46            self['file'].seek(0, 0)
     47            self['content']=self['file'].read(size)
     48        return dict.__getitem__(self, name)
     49
     50    def __deepcopy__(self, memo={}):
     51        self['content'] # make sure file content is loaded
     52        import copy
     53        result = self.__class__()
     54        memo[id(self)] = result
     55        for key, value in dict.items(self):
     56            dict.__setitem__(result, copy.deepcopy(key, memo), copy.deepcopy(value, memo))
     57        return result
     58
     59class FieldStorage(cgi.FieldStorage):
     60    "cgi.FieldStorage with ability to store files on disk or in memory"
     61    def make_file(self, binary=None):
     62        return cgi.FieldStorage.make_file(self, binary)   
     63
     64    def read_lines_to_eof(self):
     65        """Internal: read lines until EOF."""
     66        while 1:
     67            line = self.fp.readline(64000)
     68            if not line:
     69                self.done = -1
     70                break
     71            self.__write(line)
     72
     73    def read_lines_to_outerboundary(self):
     74        """Internal: read lines until outerboundary."""
     75        next = "--" + self.outerboundary
     76        last = next + "--"
     77        delim = ""
     78        while 1:
     79            line = self.fp.readline(64000)
     80            if not line:
     81                self.done = -1
     82                break
     83            if line[:2] == "--":
     84                strippedline = line.strip()
     85                if strippedline == next:
     86                    break
     87                if strippedline == last:
     88                    self.done = 1
     89                    break
     90            odelim = delim
     91            if line[-2:] == "\r\n":
     92                delim = "\r\n"
     93                line = line[:-2]
     94            elif line[-1] == "\n":
     95                delim = "\n"
     96                line = line[:-1]
     97            else:
     98                delim = ""
     99            self.__write(odelim + line)
     100
     101
     102
     103def parse_file_upload(post_stream, environ):
    39104    "Returns a tuple of (POST MultiValueDict, FILES MultiValueDict)"
    40     import email, email.Message
    41     from cgi import parse_header
    42     raw_message = '\r\n'.join(['%s:%s' % pair for pair in header_dict.items()])
    43     raw_message += '\r\n\r\n' + post_data
    44     msg = email.message_from_string(raw_message)
     105    fs = FieldStorage(post_stream, environ=environ)
    45106    POST = MultiValueDict()
    46107    FILES = MultiValueDict()
    47     for submessage in msg.get_payload():
    48         if isinstance(submessage, email.Message.Message):
    49             name_dict = parse_header(submessage['Content-Disposition'])[1]
    50             # name_dict is something like {'name': 'file', 'filename': 'test.txt'} for file uploads
    51             # or {'name': 'blah'} for POST fields
    52             # We assume all uploaded files have a 'filename' set.
    53             if name_dict.has_key('filename'):
    54                 assert type([]) != type(submessage.get_payload()), "Nested MIME messages are not supported"
    55                 if not name_dict['filename'].strip():
     108    for key in fs.keys():
     109        # We can't use FieldStorage.getlist to get contents of a
     110        # field as a list because for file fields it returns only filenames
     111        if type(fs[key]) == type([]):
     112            field_list = fs[key]
     113        else:
     114            field_list = [fs[key]]
     115        for field in field_list:
     116            if hasattr(field, 'filename') and field.filename is not None:
     117                if not field.filename.strip():
    56118                    continue
    57119                # IE submits the full path, so trim everything but the basename.
    58120                # (We can't use os.path.basename because it expects Linux paths.)
    59                 filename = name_dict['filename'][name_dict['filename'].rfind("\\")+1:]
    60                 FILES.appendlist(name_dict['name'], {
     121                filename = field.filename[field.filename.rfind("\\") + 1:]
     122                FILES.appendlist(key, FileDict({
    61123                    'filename': filename,
    62                     'content-type': (submessage.has_key('Content-Type') and submessage['Content-Type'] or None),
    63                     'content': submessage.get_payload(),
    64                 })
     124                    'content-type': field.type,
     125                    'file': field.file,
     126                }))
    65127            else:
    66                 POST.appendlist(name_dict['name'], submessage.get_payload())
     128                POST.appendlist(key, field.value)
    67129    return POST, FILES
    68130
    69131class QueryDict(MultiValueDict):
  • django/conf/global_settings.py

     
    211211# Hint: you really don't!
    212212TRANSACTIONS_MANAGED = False
    213213
     214# Whether to store uploaded files in temp files rather than in memory.
     215# Storing files on disk may be necessary for accepting large files.
     216STORE_UPLOAD_ON_DISK = False
     217
    214218##############
    215219# MIDDLEWARE #
    216220##############
  • django/core/handlers/wsgi.py

     
    6969        # Populates self._post and self._files
    7070        if self.environ['REQUEST_METHOD'] == 'POST':
    7171            if self.environ.get('CONTENT_TYPE', '').startswith('multipart'):
    72                 header_dict = dict([(k, v) for k, v in self.environ.items() if k.startswith('HTTP_')])
    73                 header_dict['Content-Type'] = self.environ.get('CONTENT_TYPE', '')
    74                 self._post, self._files = http.parse_file_upload(header_dict, self.raw_post_data)
     72                self._post, self._files = http.parse_file_upload(self.environ['wsgi.input'], self.environ)
    7573            else:
    7674                self._post, self._files = http.QueryDict(self.raw_post_data), datastructures.MultiValueDict()
    7775        else:
  • django/core/handlers/modpython.py

     
    2626    def _load_post_and_files(self):
    2727        "Populates self._post and self._files"
    2828        if self._req.headers_in.has_key('content-type') and self._req.headers_in['content-type'].startswith('multipart'):
    29             self._post, self._files = http.parse_file_upload(self._req.headers_in, self.raw_post_data)
     29            environ = dict(self.META)
     30            environ['CONTENT_LENGTH'] = environ['HTTP_CONTENT_LENGTH']
     31            environ['CONTENT_TYPE'] = environ['HTTP_CONTENT_TYPE']
     32            self._post, self._files = http.parse_file_upload(self._req, environ)
    3033        else:
    3134            self._post, self._files = http.QueryDict(self.raw_post_data), datastructures.MultiValueDict()
    3235
Back to Top