Ticket #1484: 3013-streaming-file_rfc822.diff

File 3013-streaming-file_rfc822.diff, 13.1 KB (added by [530], 9 years ago)

Fixed readline in rfc822

  • django/http/__init__.py

     
    22from pprint import pformat
    33from urllib import urlencode
    44from django.utils.datastructures import MultiValueDict
     5import cgi
     6import rfc822
    57
     8from StringIO import StringIO
     9
    610try:
    711    # The mod_python version is more efficient, so try importing it first.
    812    from mod_python.util import parse_qsl
     
    3539    def get_full_path(self):
    3640        return ''
    3741
    38 def parse_file_upload(header_dict, post_data):
     42class Message(rfc822.Message):
     43
     44    def readheaders(self):
     45        """Read header lines. extended class to fix readline"""
     46
     47        self.dict = {}
     48        self.unixfrom = ''
     49        self.headers = list = []
     50        self.status = ''
     51        headerseen = ""
     52        firstline = 1
     53        startofline = unread = tell = None
     54        if hasattr(self.fp, 'unread'):
     55            unread = self.fp.unread
     56        elif self.seekable:
     57            tell = self.fp.tell
     58        while 1:
     59            if tell:
     60                try:
     61                    startofline = tell()
     62                except IOError:
     63                    startofline = tell = None
     64                    self.seekable = 0
     65            line = self.fp.readline(64000)
     66            if not line:
     67                self.status = 'EOF in headers'
     68                break
     69            # Skip unix From name time lines
     70            if firstline and line.startswith('From '):
     71                self.unixfrom = self.unixfrom + line
     72                continue
     73            firstline = 0
     74            if headerseen and line[0] in ' \t':
     75                # It's a continuation line.
     76                list.append(line)
     77                x = (self.dict[headerseen] + "\n " + line.strip())
     78                self.dict[headerseen] = x.strip()
     79                continue
     80            elif self.iscomment(line):
     81                # It's a comment.  Ignore it.
     82                continue
     83            elif self.islast(line):
     84                # Note! No pushback here!  The delimiter line gets eaten.
     85                break
     86            headerseen = self.isheader(line)
     87            if headerseen:
     88                # It's a legal header line, save it.
     89                list.append(line)
     90                self.dict[headerseen] = line[len(headerseen)+1:].strip()
     91                continue
     92            else:
     93                # It's not a header line; throw it back and stop here.
     94                if not self.dict:
     95                    self.status = 'No headers'
     96                else:
     97                    self.status = 'Non-header line where header expected'
     98                # Try to undo the read.
     99                if unread:
     100                    unread(line)
     101                elif tell:
     102                    self.fp.seek(startofline)
     103                else:
     104                    self.status = self.status + '; bad seek'
     105                break
     106
     107
     108class FileDict(dict):
     109    "Keeps uploaded file as a file-like object and reads its content on demand"
     110    def __getitem__(self, name):
     111        if name=='content' and not 'content' in self:
     112            self['file'].seek(0, 2)
     113            size = self['file'].tell()
     114            self['file'].seek(0, 0)
     115            self['content']=self['file'].read(size)
     116        return dict.__getitem__(self, name)
     117   
     118    def get_size(self):
     119        self['file'].seek(0, 2)   
     120        size = self['file'].tell()
     121        return size
     122 
     123    def __repr__(self):
     124        return '<FileDict>'
     125
     126    def __deepcopy__(self, memo={}):
     127        self['content'] # make sure file content is loaded
     128        import copy
     129        result = self.__class__()
     130        memo[id(self)] = result
     131        for key, value in dict.items(self):
     132            dict.__setitem__(result, copy.deepcopy(key, memo), copy.deepcopy(value, memo))
     133        return result
     134
     135class FieldStorage(cgi.FieldStorage):
     136    "cgi.FieldStorage with ability to store files on disk"
     137    def make_file(self, binary=None):
     138   
     139        import tempfile
     140        tmpfile = tempfile.NamedTemporaryFile("w+b")
     141        self.tmp_name = tmpfile.name
     142        return tmpfile
     143
     144    def read_multi(self, environ, keep_blank_values, strict_parsing):
     145        """Internal: read a part that is itself multipart."""
     146        ib = self.innerboundary
     147        if not cgi.valid_boundary(ib):
     148            raise ValueError, 'Invalid boundary in multipart form: %r' % (ib,)
     149        self.list = []
     150        klass = self.FieldStorageClass or self.__class__
     151        part = klass(self.fp, {}, ib,
     152                     environ, keep_blank_values, strict_parsing)
     153        # Throw first part away
     154        while not part.done:
     155            headers = Message(self.fp)
     156            part = klass(self.fp, headers, ib,
     157                         environ, keep_blank_values, strict_parsing)
     158            self.list.append(part)
     159        self.skip_lines()
     160
     161    def read_lines_to_eof(self):
     162        """Internal: read lines until EOF."""
     163        while 1:
     164            line = self.fp.readline(64000) # chunked read
     165            if not line:
     166                self.done = -1
     167                break
     168            self.__write(line)
     169
     170    def read_lines_to_outerboundary(self):
     171        """Internal: read lines until outerboundary."""
     172        next = "--" + self.outerboundary
     173        last = next + "--"
     174        delim = ""
     175        while 1:
     176            line = self.fp.readline(64000) # chunked read
     177            if not line:
     178                self.done = -1
     179                break
     180            if line[:2] == "--":
     181                strippedline = line.strip()
     182                if strippedline == next:
     183                    break
     184                if strippedline == last:
     185                    self.done = 1
     186                    break
     187            odelim = delim
     188            if line[-2:] == "\r\n":
     189                delim = "\r\n"
     190                line = line[:-2]
     191            elif line[-1] == "\n":
     192                delim = "\n"
     193                line = line[:-1]
     194            else:
     195                delim = ""
     196            self.__write(odelim + line)
     197
     198
     199
     200def parse_file_upload(post_stream, environ):
    39201    "Returns a tuple of (POST MultiValueDict, FILES MultiValueDict)"
    40     import email, email.Message
    41     from cgi import parse_header
    42     raw_message = '\r\n'.join(['%s:%s' % pair for pair in header_dict.items()])
    43     raw_message += '\r\n\r\n' + post_data
    44     msg = email.message_from_string(raw_message)
     202    fs = FieldStorage(post_stream, environ=environ)
    45203    POST = MultiValueDict()
    46204    FILES = MultiValueDict()
    47     for submessage in msg.get_payload():
    48         if isinstance(submessage, email.Message.Message):
    49             name_dict = parse_header(submessage['Content-Disposition'])[1]
    50             # name_dict is something like {'name': 'file', 'filename': 'test.txt'} for file uploads
    51             # or {'name': 'blah'} for POST fields
    52             # We assume all uploaded files have a 'filename' set.
    53             if name_dict.has_key('filename'):
    54                 assert type([]) != type(submessage.get_payload()), "Nested MIME messages are not supported"
    55                 if not name_dict['filename'].strip():
     205    for key in fs.keys():
     206        # We can't use FieldStorage.getlist to get contents of a
     207        # field as a list because for file fields it returns only filenames
     208        if type(fs[key]) == type([]):
     209            field_list = fs[key]
     210        else:
     211            field_list = [fs[key]]
     212        for field in field_list:
     213            if hasattr(field, 'filename') and field.filename is not None:
     214                if not field.filename.strip():
    56215                    continue
    57216                # IE submits the full path, so trim everything but the basename.
    58217                # (We can't use os.path.basename because it expects Linux paths.)
    59                 filename = name_dict['filename'][name_dict['filename'].rfind("\\")+1:]
    60                 FILES.appendlist(name_dict['name'], {
     218                filename = field.filename[field.filename.rfind("\\") + 1:]
     219                FILES.appendlist(key, FileDict({
    61220                    'filename': filename,
    62                     'content-type': (submessage.has_key('Content-Type') and submessage['Content-Type'] or None),
    63                     'content': submessage.get_payload(),
    64                 })
     221                    'content-type': field.type,
     222                    'file': field.file,
     223                    'tmp_name': field.tmp_name
     224                }))
    65225            else:
    66                 POST.appendlist(name_dict['name'], submessage.get_payload())
     226                POST.appendlist(key, field.value)
    67227    return POST, FILES
    68228
    69229class QueryDict(MultiValueDict):
  • django/db/models/base.py

     
    300300    def _get_FIELD_size(self, field):
    301301        return os.path.getsize(self._get_FIELD_filename(field))
    302302
    303     def _save_FIELD_file(self, field, filename, raw_contents):
     303    def _save_FIELD_file(self, field, filename, temp_file):
    304304        directory = field.get_directory_name()
    305305        try: # Create the date-based directory if it doesn't exist.
    306306            os.makedirs(os.path.join(settings.MEDIA_ROOT, directory))
     
    322322        setattr(self, field.attname, filename)
    323323
    324324        full_filename = self._get_FIELD_filename(field)
    325         fp = open(full_filename, 'wb')
    326         fp.write(raw_contents)
    327         fp.close()
     325        #fp = open(full_filename, 'wb')
     326        #fp.write(raw_contents)
     327        #fp.close()
    328328
     329        # move file
     330        os.rename(temp_file, full_filename)
     331
    329332        # Save the width and/or height, if applicable.
    330333        if isinstance(field, ImageField) and (field.width_field or field.height_field):
    331334            from django.utils.images import get_image_dimensions
  • django/db/models/fields/__init__.py

     
    594594        if new_data.get(upload_field_name, False):
    595595            func = getattr(new_object, 'save_%s_file' % self.name)
    596596            if rel:
    597                 func(new_data[upload_field_name][0]["filename"], new_data[upload_field_name][0]["content"])
     597                func(new_data[upload_field_name][0]["filename"], new_data[upload_field_name][0]["tmp_name"])
    598598            else:
    599                 func(new_data[upload_field_name]["filename"], new_data[upload_field_name]["content"])
     599                func(new_data[upload_field_name]["filename"], new_data[upload_field_name]["tmp_name"])
    600600
    601601    def get_directory_name(self):
    602602        return os.path.normpath(datetime.datetime.now().strftime(self.upload_to))
  • django/forms/__init__.py

     
    641641        self.validator_list = [self.isNonEmptyFile] + validator_list
    642642
    643643    def isNonEmptyFile(self, field_data, all_data):
    644         if not field_data['content']:
     644        if field_data.get_size()<1:
    645645            raise validators.CriticalValidationError, gettext("The submitted file is empty.")
    646646
    647647    def render(self, data):
  • django/core/handlers/wsgi.py

     
    6969        # Populates self._post and self._files
    7070        if self.environ['REQUEST_METHOD'] == 'POST':
    7171            if self.environ.get('CONTENT_TYPE', '').startswith('multipart'):
    72                 header_dict = dict([(k, v) for k, v in self.environ.items() if k.startswith('HTTP_')])
    73                 header_dict['Content-Type'] = self.environ.get('CONTENT_TYPE', '')
    74                 self._post, self._files = http.parse_file_upload(header_dict, self.raw_post_data)
     72                self._post, self._files = http.parse_file_upload(self.environ['wsgi.input'], self.environ)
    7573            else:
    7674                self._post, self._files = http.QueryDict(self.raw_post_data), datastructures.MultiValueDict()
    7775        else:
  • django/core/handlers/modpython.py

     
    2626    def _load_post_and_files(self):
    2727        "Populates self._post and self._files"
    2828        if self._req.headers_in.has_key('content-type') and self._req.headers_in['content-type'].startswith('multipart'):
    29             self._post, self._files = http.parse_file_upload(self._req.headers_in, self.raw_post_data)
     29            environ = dict(self.META)
     30            environ['CONTENT_LENGTH'] = environ['HTTP_CONTENT_LENGTH']
     31            environ['CONTENT_TYPE'] = environ['HTTP_CONTENT_TYPE']
     32            self._post, self._files = http.parse_file_upload(self._req, environ)
    3033        else:
    3134            self._post, self._files = http.QueryDict(self.raw_post_data), datastructures.MultiValueDict()
    3235
Back to Top