Ticket #1484: 3013-streaming-file_improved.diff

File 3013-streaming-file_improved.diff, 10.4 KB (added by [530], 18 years ago)

Improved

  • django/http/__init__.py

     
    22from pprint import pformat
    33from urllib import urlencode
    44from django.utils.datastructures import MultiValueDict
     5import cgi
     6from StringIO import StringIO
    57
    68try:
    79    # The mod_python version is more efficient, so try importing it first.
    … …  
    3537    def get_full_path(self):
    3638        return ''
    3739
    38 def parse_file_upload(header_dict, post_data):
     40class FileDict(dict):
     41    "Keeps uploaded file as a file-like object and reads its content on demand"
     42    def __getitem__(self, name):
     43        if name=='content' and not 'content' in self:
     44            self['file'].seek(0, 2)
     45            size = self['file'].tell()
     46            self['file'].seek(0, 0)
     47            self['content']=self['file'].read(size)
     48        return dict.__getitem__(self, name)
     49   
     50    def get_size(self):
     51        self['file'].seek(0, 2)   
     52        size = self['file'].tell()
     53        return size
     54 
     55    def __repr__(self):
     56        return '<FileDict>'
     57
     58    def __deepcopy__(self, memo={}):
     59        self['content'] # make sure file content is loaded
     60        import copy
     61        result = self.__class__()
     62        memo[id(self)] = result
     63        for key, value in dict.items(self):
     64            dict.__setitem__(result, copy.deepcopy(key, memo), copy.deepcopy(value, memo))
     65        return result
     66
     67class FieldStorage(cgi.FieldStorage):
     68    "cgi.FieldStorage with ability to store files on disk"
     69    def make_file(self, binary=None):
     70   
     71        import tempfile
     72        tmpfile = tempfile.NamedTemporaryFile("w+b")
     73        self.tmp_name = tmpfile.name
     74        return tmpfile
     75
     76    def read_lines_to_eof(self):
     77        """Internal: read lines until EOF."""
     78        while 1:
     79            line = self.fp.readline(64000) # chunked read
     80            if not line:
     81                self.done = -1
     82                break
     83            self.__write(line)
     84
     85    def read_lines_to_outerboundary(self):
     86        """Internal: read lines until outerboundary."""
     87        next = "--" + self.outerboundary
     88        last = next + "--"
     89        delim = ""
     90        while 1:
     91            line = self.fp.readline(64000) # chunked read
     92            if not line:
     93                self.done = -1
     94                break
     95            if line[:2] == "--":
     96                strippedline = line.strip()
     97                if strippedline == next:
     98                    break
     99                if strippedline == last:
     100                    self.done = 1
     101                    break
     102            odelim = delim
     103            if line[-2:] == "\r\n":
     104                delim = "\r\n"
     105                line = line[:-2]
     106            elif line[-1] == "\n":
     107                delim = "\n"
     108                line = line[:-1]
     109            else:
     110                delim = ""
     111            self.__write(odelim + line)
     112
     113
     114
     115def parse_file_upload(post_stream, environ):
    39116    "Returns a tuple of (POST MultiValueDict, FILES MultiValueDict)"
    40     import email, email.Message
    41     from cgi import parse_header
    42     raw_message = '\r\n'.join(['%s:%s' % pair for pair in header_dict.items()])
    43     raw_message += '\r\n\r\n' + post_data
    44     msg = email.message_from_string(raw_message)
     117    fs = FieldStorage(post_stream, environ=environ)
    45118    POST = MultiValueDict()
    46119    FILES = MultiValueDict()
    47     for submessage in msg.get_payload():
    48         if isinstance(submessage, email.Message.Message):
    49             name_dict = parse_header(submessage['Content-Disposition'])[1]
    50             # name_dict is something like {'name': 'file', 'filename': 'test.txt'} for file uploads
    51             # or {'name': 'blah'} for POST fields
    52             # We assume all uploaded files have a 'filename' set.
    53             if name_dict.has_key('filename'):
    54                 assert type([]) != type(submessage.get_payload()), "Nested MIME messages are not supported"
    55                 if not name_dict['filename'].strip():
     120    for key in fs.keys():
     121        # We can't use FieldStorage.getlist to get contents of a
     122        # field as a list because for file fields it returns only filenames
     123        if type(fs[key]) == type([]):
     124            field_list = fs[key]
     125        else:
     126            field_list = [fs[key]]
     127        for field in field_list:
     128            if hasattr(field, 'filename') and field.filename is not None:
     129                if not field.filename.strip():
    56130                    continue
    57131                # IE submits the full path, so trim everything but the basename.
    58132                # (We can't use os.path.basename because it expects Linux paths.)
    59                 filename = name_dict['filename'][name_dict['filename'].rfind("\\")+1:]
    60                 FILES.appendlist(name_dict['name'], {
     133                filename = field.filename[field.filename.rfind("\\") + 1:]
     134                FILES.appendlist(key, FileDict({
    61135                    'filename': filename,
    62                     'content-type': (submessage.has_key('Content-Type') and submessage['Content-Type'] or None),
    63                     'content': submessage.get_payload(),
    64                 })
     136                    'content-type': field.type,
     137                    'file': field.file,
     138                    'tmp_name': field.tmp_name
     139                }))
    65140            else:
    66                 POST.appendlist(name_dict['name'], submessage.get_payload())
     141                POST.appendlist(key, field.value)
    67142    return POST, FILES
    68143
    69144class QueryDict(MultiValueDict):

  • django/conf/global_settings.py

     
    211211# Hint: you really don't!
    212212TRANSACTIONS_MANAGED = False
    213213
     214# Whether to store uploaded files in temp files rather than in memory.
     215# Storing files on disk may be necessary for accepting large files.
     216STORE_UPLOAD_ON_DISK = False
     217
    214218##############
    215219# MIDDLEWARE #
    216220##############

  • django/db/models/base.py

     
    300300    def _get_FIELD_size(self, field):
    301301        return os.path.getsize(self._get_FIELD_filename(field))
    302302
    303     def _save_FIELD_file(self, field, filename, raw_contents):
     303    def _save_FIELD_file(self, field, filename, temp_file):
    304304        directory = field.get_directory_name()
    305305        try: # Create the date-based directory if it doesn't exist.
    306306            os.makedirs(os.path.join(settings.MEDIA_ROOT, directory))
    … …  
    322322        setattr(self, field.attname, filename)
    323323
    324324        full_filename = self._get_FIELD_filename(field)
    325         fp = open(full_filename, 'wb')
    326         fp.write(raw_contents)
    327         fp.close()
     325        #fp = open(full_filename, 'wb')
     326        #fp.write(raw_contents)
     327        #fp.close()
    328328
     329        # move file
     330        os.rename(temp_file, full_filename)
     331
    329332        # Save the width and/or height, if applicable.
    330333        if isinstance(field, ImageField) and (field.width_field or field.height_field):
    331334            from django.utils.images import get_image_dimensions

  • django/db/models/fields/__init__.py

     
    594594        if new_data.get(upload_field_name, False):
    595595            func = getattr(new_object, 'save_%s_file' % self.name)
    596596            if rel:
    597                 func(new_data[upload_field_name][0]["filename"], new_data[upload_field_name][0]["content"])
     597                func(new_data[upload_field_name][0]["filename"], new_data[upload_field_name][0]["tmp_name"])
    598598            else:
    599                 func(new_data[upload_field_name]["filename"], new_data[upload_field_name]["content"])
     599                func(new_data[upload_field_name]["filename"], new_data[upload_field_name]["tmp_name"])
    600600
    601601    def get_directory_name(self):
    602602        return os.path.normpath(datetime.datetime.now().strftime(self.upload_to))

  • django/forms/__init__.py

     
    641641        self.validator_list = [self.isNonEmptyFile] + validator_list
    642642
    643643    def isNonEmptyFile(self, field_data, all_data):
    644         if not field_data['content']:
     644        if field_data.get_size()<1:
    645645            raise validators.CriticalValidationError, gettext("The submitted file is empty.")
    646646
    647647    def render(self, data):

  • django/core/handlers/wsgi.py

     
    6969        # Populates self._post and self._files
    7070        if self.environ['REQUEST_METHOD'] == 'POST':
    7171            if self.environ.get('CONTENT_TYPE', '').startswith('multipart'):
    72                 header_dict = dict([(k, v) for k, v in self.environ.items() if k.startswith('HTTP_')])
    73                 header_dict['Content-Type'] = self.environ.get('CONTENT_TYPE', '')
    74                 self._post, self._files = http.parse_file_upload(header_dict, self.raw_post_data)
     72                self._post, self._files = http.parse_file_upload(self.environ['wsgi.input'], self.environ)
    7573            else:
    7674                self._post, self._files = http.QueryDict(self.raw_post_data), datastructures.MultiValueDict()
    7775        else:

  • django/core/handlers/modpython.py

     
    2626    def _load_post_and_files(self):
    2727        "Populates self._post and self._files"
    2828        if self._req.headers_in.has_key('content-type') and self._req.headers_in['content-type'].startswith('multipart'):
    29             self._post, self._files = http.parse_file_upload(self._req.headers_in, self.raw_post_data)
     29            environ = dict(self.META)
     30            environ['CONTENT_LENGTH'] = environ['HTTP_CONTENT_LENGTH']
     31            environ['CONTENT_TYPE'] = environ['HTTP_CONTENT_TYPE']
     32            self._post, self._files = http.parse_file_upload(self._req, environ)
    3033        else:
    3134            self._post, self._files = http.QueryDict(self.raw_post_data), datastructures.MultiValueDict()
    3235
Back to Top