Ticket #2070: 5113_file_upload_core_with_middleware_hooks.diff

File 5113_file_upload_core_with_middleware_hooks.diff, 39.1 KB (added by Michael Axiak <axiak@…>, 8 years ago)

Fixes WSGI bug.

  • django/http/file_descriptor.py

     
     1"""
     2This file contains a fallback FileProgressDescriptor
     3for file upload progress.
     4"""
     5import pickle
     6import os
     7
     8class DefaultFileProgressDescriptor(object):
     9
     10    def __init__(self, FileException):
     11        self.FileException = FileException
     12   
     13    def __get__(self, request, HttpRequest):
     14        """
     15        Returns the file progress for this request.
     16        If no file progress is known, returns an empty
     17        dictionary.
     18        The request also keeps a local copy so that
     19        the file is not accessed every time one wants to
     20        ask for something.
     21        """
     22        from django.conf import settings
     23
     24        file_upload_dir = settings.FILE_UPLOAD_DIR
     25        progress_id     = request.META['UPLOAD_PROGRESS_ID']
     26       
     27        if not progress_id or not file_upload_dir:
     28            return {'wow': 'true'}
     29            return {}
     30
     31        if getattr(self, '_file_progress', False) != False:
     32            return self._file_progress
     33
     34        try:
     35            f = open(os.path.join(file_upload_dir, progress_id), 'rb')
     36            progress = pickle.load(f)
     37            f.close()
     38            self._file_progress = progress
     39            return progress
     40        except:
     41            self._file_progress = {}
     42            return {}
     43
     44    def __set__(self, request, new_progress):
     45        """
     46        Sets the value of the file progress for this request.
     47        If no file progress is underway, raises an error.
     48        """
     49
     50        from django.conf import settings
     51
     52        file_upload_dir = settings.FILE_UPLOAD_DIR
     53        progress_id     = request.META['UPLOAD_PROGRESS_ID']
     54
     55        if not progress_id or not file_upload_dir:
     56            raise self.FileException('There is no upload in progress.')
     57
     58        self._file_progress = new_progress
     59        f = open(os.path.join(file_upload_dir, progress_id), 'wb')
     60        pickle.dump(new_progress, f)
     61        f.close()
     62
     63    def __delete__(self, request):
     64        """
     65        Removes the file if there is an upload in process.
     66        """
     67        file_upload_dir = settings.FILE_UPLOAD_DIR
     68        progress_id     = request.META['UPLOAD_PROGRESS_ID']
     69
     70        if not progress_id or not file_upload_dir:
     71            raise self.FileException('There is no upload in progress.')
     72
     73        try:
     74            os.remove(os.path.join(file_upload_dir, progress_id))
     75        except:
     76            pass
  • django/http/__init__.py

     
    1 import os
     1import os, pickle
    22from Cookie import SimpleCookie
    33from pprint import pformat
    44from urllib import urlencode, quote
    55from django.utils.datastructures import MultiValueDict
     6from django.http.file_descriptor import DefaultFileProgressDescriptor
     7import re
    68
     9try:
     10    from cStringIO import StringIO
     11except ImportError:
     12    from StringIO import StringIO
     13
    714RESERVED_CHARS="!*'();:@&=+$,/?%#[]"
    815
     16
    917try:
    1018    # The mod_python version is more efficient, so try importing it first.
    1119    from mod_python.util import parse_qsl
    1220except ImportError:
    1321    from cgi import parse_qsl
    1422
     23class MetaFileProgressDescriptor(object):
     24    """
     25    This descriptor allows other descriptors to
     26    be loaded in runtime to a request instance.
     27    """
     28    def __get__(self, request, *args, **kwargs):
     29        return request._file_progress.__get__(request, *args, **kwargs)
     30
     31    def __set__(self, request, *args, **kwargs):
     32        return request._file_progress.__set__(request, *args, **kwargs)
     33
     34    def __delete__(self, request, *args, **kwargs):
     35        return request._file_progress.__delete__(request, *args, **kwargs)
     36
    1537class Http404(Exception):
    1638    pass
    1739
    1840class HttpRequest(object):
    1941    "A basic HTTP request"
     42
     43    upload_id_re = re.compile(r'^[a-fA-F0-9]{32}$')
     44    file_progress = MetaFileProgressDescriptor()
     45
    2046    def __init__(self):
    2147        self.GET, self.POST, self.COOKIES, self.META, self.FILES = {}, {}, {}, {}, {}
    2248        self.path = ''
     
    4167
    4268    def is_secure(self):
    4369        return os.environ.get("HTTPS") == "on"
     70 
     71    def _get_file_progress_from_args(self, headers, get, querystring):
    4472
    45 def parse_file_upload(header_dict, post_data):
    46     "Returns a tuple of (POST MultiValueDict, FILES MultiValueDict)"
    47     import email, email.Message
    48     from cgi import parse_header
    49     raw_message = '\r\n'.join(['%s:%s' % pair for pair in header_dict.items()])
    50     raw_message += '\r\n\r\n' + post_data
    51     msg = email.message_from_string(raw_message)
    52     POST = MultiValueDict()
    53     FILES = MultiValueDict()
    54     for submessage in msg.get_payload():
    55         if submessage and isinstance(submessage, email.Message.Message):
    56             name_dict = parse_header(submessage['Content-Disposition'])[1]
    57             # name_dict is something like {'name': 'file', 'filename': 'test.txt'} for file uploads
    58             # or {'name': 'blah'} for POST fields
    59             # We assume all uploaded files have a 'filename' set.
    60             if 'filename' in name_dict:
    61                 assert type([]) != type(submessage.get_payload()), "Nested MIME messages are not supported"
    62                 if not name_dict['filename'].strip():
    63                     continue
    64                 # IE submits the full path, so trim everything but the basename.
    65                 # (We can't use os.path.basename because it expects Linux paths.)
    66                 filename = name_dict['filename'][name_dict['filename'].rfind("\\")+1:]
    67                 FILES.appendlist(name_dict['name'], {
    68                     'filename': filename,
    69                     'content-type': 'Content-Type' in submessage and submessage['Content-Type'] or None,
    70                     'content': submessage.get_payload(),
    71                 })
    72             else:
    73                 POST.appendlist(name_dict['name'], submessage.get_payload())
    74     return POST, FILES
     73        if 'X-Upload-ID' in headers:
     74            progress_id = headers['X-Upload-ID']
     75        elif 'X-Progress-ID' in headers:
     76            progress_id = headers['X-Progress-ID']
     77        elif 'upload_id' in get:
     78            progress_id = get['upload_id']
     79        elif 'progress_id' in get:
     80            progress_id = get['progress_id']
     81        elif querystring != None and len(querystring.strip()) == 32:
     82            progress_id = querystring
     83        else:
     84            return None
    7585
     86        if not self.upload_id_re.match(progress_id):
     87            return None
     88
     89        return progress_id
     90
     91
     92def parse_file_upload(headers, input, request):
     93    from django.conf import settings
     94
     95    # Only stream files to disk if FILE_STREAMING_DIR is set
     96    file_upload_dir = settings.FILE_UPLOAD_DIR
     97    streaming_min_post_size = settings.STREAMING_MIN_POST_SIZE
     98
     99    try:
     100        parser = MultiPartParser(headers, input, request, file_upload_dir, streaming_min_post_size)
     101        return parser.parse()
     102    except MultiPartParserError, e:
     103        return MultiValueDict({ '_file_upload_error': [e.message] }), {}
     104
     105class MultiPartParserError(Exception):
     106    def __init__(self, message):
     107        self.message = message
     108    def __str__(self):
     109        return repr(self.message)
     110       
     111class MultiPartParser(object):
     112    """
     113    A rfc2388 multipart/form-data parser.
     114   
     115    parse() reads the input stream in chunk_size chunks and returns a
     116    tuple of (POST MultiValueDict, FILES MultiValueDict). If
     117    file_upload_dir is defined files will be streamed to temporary
     118    files in the specified directory.
     119
     120    The FILES dictionary will have 'filename', 'content-type',
     121    'content' and 'content-length' entries. For streamed files it will
     122    also have 'tmpfilename' and 'tmpfile'. The 'content' entry will
     123    only be read from disk when referenced for streamed files.
     124
     125    If the header X-Progress-ID is sent with a 32 character hex string
     126    a temporary file with the same name will be created in
     127    `file_upload_dir`` with a pickled { 'received', 'size' }
     128    dictionary with the number of bytes received and the size expected
     129    respectively. The file will be unlinked when the parser finishes.
     130
     131    """
     132
     133    def __init__(self, headers, input, request, file_upload_dir=None, streaming_min_post_size=None, chunk_size=1024*64):
     134        try:
     135            content_length = int(headers['Content-Length'])
     136        except:
     137            raise MultiPartParserError('Invalid Content-Length: %s' % headers.get('Content-Length'))
     138
     139        content_type = headers.get('Content-Type')
     140
     141        if not content_type or not content_type.startswith('multipart/'):
     142            raise MultiPartParserError('Invalid Content-Type: %s' % content_type)
     143           
     144        ctype, opts = self.parse_header(content_type)
     145        boundary = opts.get('boundary')
     146        from cgi import valid_boundary
     147        if not boundary or not valid_boundary(boundary):
     148            raise MultiPartParserError('Invalid boundary in multipart form: %s' % boundary)
     149
     150        progress_id = request.META['UPLOAD_PROGRESS_ID']
     151
     152        if file_upload_dir and progress_id:
     153            self._progress_filename = os.path.join(file_upload_dir, progress_id)
     154        else:
     155            self._progress_filename = None
     156        self._boundary = '--' + boundary
     157        self._input = input
     158        self._size = content_length
     159        self._received = 0
     160        self._file_upload_dir = file_upload_dir
     161        self._chunk_size = chunk_size
     162        self._state = 'PREAMBLE'
     163        self._partial = ''
     164        self._post = MultiValueDict()
     165        self._files = MultiValueDict()
     166        self._request = request
     167       
     168        if streaming_min_post_size is not None and content_length < streaming_min_post_size:
     169            self._file_upload_dir = None # disable file streaming for small request
     170
     171        try:
     172            # use mx fast string search if available
     173            from mx.TextTools import FS
     174            self._fs = FS(self._boundary)
     175        except ImportError:
     176            self._fs = None
     177
     178    def parse(self):
     179        try:
     180            self._parse()
     181        finally:
     182            if self._progress_filename:
     183                self._request.file_progress = {'state': 'done'}
     184               
     185       
     186        return self._post, self._files
     187
     188    def _parse(self):
     189        size = self._size
     190
     191        try:
     192            while size > 0:
     193                n = self._read(self._input, min(self._chunk_size, size))
     194                if not n:
     195                    break
     196                size -= n
     197        except:
     198            # consume any remaining data so we dont generate a "Connection Reset" error
     199            size = self._size - self._received
     200            while size > 0:
     201                data = self._input.read(min(self._chunk_size, size))
     202                size -= len(data)
     203            raise
     204
     205    def _find_boundary(self, data, start, stop):
     206        """
     207        Find the next boundary and return the end of current part
     208        and start of next part.
     209        """
     210        if self._fs:
     211            boundary = self._fs.find(data, start, stop)
     212        else:
     213            boundary = data.find(self._boundary, start, stop)
     214        if boundary >= 0:
     215            end = boundary
     216            next = boundary + len(self._boundary)
     217
     218            # backup over CRLF
     219            if end > 0 and data[end-1] == '\n': end -= 1
     220            if end > 0 and data[end-1] == '\r': end -= 1
     221            # skip over --CRLF
     222            if next < stop and data[next] == '-': next += 1
     223            if next < stop and data[next] == '-': next += 1
     224            if next < stop and data[next] == '\r': next += 1
     225            if next < stop and data[next] == '\n': next += 1
     226
     227            return True, end, next
     228        else:
     229            return False, stop, stop
     230
     231    class TemporaryFile(object):
     232        "A temporary file that tries to delete itself when garbage collected."
     233        def __init__(self, dir):
     234            import tempfile
     235            (fd, name) = tempfile.mkstemp(suffix='.upload', dir=dir)
     236            self.file = os.fdopen(fd, 'w+b')
     237            self.name = name
     238
     239        def __getattr__(self, name):
     240            a = getattr(self.__dict__['file'], name)
     241            if type(a) != type(0):
     242                setattr(self, name, a)
     243            return a
     244
     245        def __del__(self):
     246            try:
     247                os.unlink(self.name)
     248            except OSError:
     249                pass
     250           
     251    class LazyContent(dict):
     252        """
     253        A lazy FILES dictionary entry that reads the contents from
     254        tmpfile only when referenced.
     255        """
     256        def __init__(self, data):
     257            dict.__init__(self, data)
     258       
     259        def __getitem__(self, key):
     260            if key == 'content' and not self.has_key(key):
     261                self['tmpfile'].seek(0)
     262                self['content'] = self['tmpfile'].read()
     263            return dict.__getitem__(self, key)
     264
     265    def _read(self, input, size):
     266        data = input.read(size)
     267
     268        if not data:
     269            return 0
     270
     271        read_size = len(data)
     272        self._received += read_size
     273
     274        if self._partial:
     275            data = self._partial + data
     276
     277        start = 0
     278        stop = len(data)
     279       
     280        while start < stop:
     281            boundary, end, next = self._find_boundary(data, start, stop)
     282
     283            if not boundary and read_size:
     284                # make sure we dont treat a partial boundary (and its separators) as data
     285                stop -= len(self._boundary) + 16
     286                end = next = stop
     287                if end <= start:
     288                    break # need more data
     289
     290            if self._state == 'PREAMBLE':
     291                # Preamble, just ignore it
     292                self._state = 'HEADER'
     293
     294            elif self._state == 'HEADER':
     295                # Beginning of header, look for end of header and parse it if found.
     296
     297                header_end = data.find('\r\n\r\n', start, stop)
     298                if header_end == -1:
     299                    break # need more data
     300
     301                header = data[start:header_end]
     302
     303                self._fieldname = None
     304                self._filename = None
     305                self._content_type = None
     306
     307                for line in header.split('\r\n'):
     308                    ctype, opts = self.parse_header(line)
     309                    if ctype == 'content-disposition: form-data':
     310                        self._fieldname = opts.get('name')
     311                        self._filename = opts.get('filename')
     312                    elif ctype.startswith('content-type: '):
     313                        self._content_type = ctype[14:]
     314
     315                if self._filename is not None:
     316                    # cleanup filename from IE full paths:
     317                    self._filename = self._filename[self._filename.rfind("\\")+1:].strip()
     318
     319                    if self._filename: # ignore files without filenames
     320                        if self._file_upload_dir:
     321                            try:
     322                                self._file = self.TemporaryFile(dir=self._file_upload_dir)
     323                            except:
     324                                raise MultiPartParserError("Failed to create temporary file.")
     325                        else:
     326                            self._file = StringIO()
     327                    else:
     328                        self._file = None
     329                    self._filesize = 0
     330                    self._state = 'FILE'
     331                else:
     332                    self._field = StringIO()
     333                    self._state = 'FIELD'
     334                next = header_end + 4
     335
     336            elif self._state == 'FIELD':
     337                # In a field, collect data until a boundary is found.
     338
     339                self._field.write(data[start:end])
     340                if boundary:
     341                    if self._fieldname:
     342                        self._post.appendlist(self._fieldname, self._field.getvalue())
     343                    self._field.close()
     344                    self._state = 'HEADER'
     345
     346            elif self._state == 'FILE':
     347                # In a file, collect data until a boundary is found.
     348
     349                if self._file:
     350                    try:
     351                        self._file.write(data[start:end])
     352                    except IOError, e:
     353                        raise MultiPartParserError("Failed to write to temporary file.")
     354                    self._filesize += end-start
     355
     356                    if self._progress_filename:
     357                        self._request.file_progress = {'received': self._received,
     358                                                       'size':     self._size,
     359                                                       'state':    'uploading'}
     360
     361                if boundary:
     362                    if self._file:
     363                        if self._file_upload_dir:
     364                            self._file.seek(0)
     365                            file = self.LazyContent({
     366                                'filename': self._filename,
     367                                'content-type':  self._content_type,
     368                                # 'content': is read on demand
     369                                'content-length': self._filesize,
     370                                'tmpfilename': self._file.name,
     371                                'tmpfile': self._file
     372                            })
     373                        else:
     374                            file = {
     375                                'filename': self._filename,
     376                                'content-type':  self._content_type,
     377                                'content': self._file.getvalue(),
     378                                'content-length': self._filesize
     379                            }
     380                            self._file.close()
     381
     382                        self._files.appendlist(self._fieldname, file)
     383
     384                    self._state = 'HEADER'
     385
     386            start = next
     387               
     388        self._partial = data[start:]
     389
     390        return read_size
     391
     392    def parse_header(self, line):
     393        from cgi import parse_header
     394        return parse_header(line)
     395
    76396class QueryDict(MultiValueDict):
    77397    """A specialized MultiValueDict that takes a query string when initialized.
    78398    This is immutable unless you create a copy of it."""
     
    306626    if not host:
    307627        host = request.META.get('HTTP_HOST', '')
    308628    return host
     629
  • django/conf/global_settings.py

     
    240240# isExistingURL validator.
    241241URL_VALIDATOR_USER_AGENT = "Django/0.96pre (http://www.djangoproject.com)"
    242242
     243# The directory to place streamed file uploads. The web server needs write
     244# permissions on this directory.
     245# If this is None, streaming uploads are disabled.
     246FILE_UPLOAD_DIR = None
     247
     248
     249# The minimum size of a POST before file uploads are streamed to disk.
     250# Any less than this number, and the file is uploaded to memory.
     251# Size is in bytes.
     252STREAMING_MIN_POST_SIZE = 512 * (2**10)
     253
     254
     255
     256
    243257##############
    244258# MIDDLEWARE #
    245259##############
     
    335349
    336350# The list of directories to search for fixtures
    337351FIXTURE_DIRS = ()
     352
     353
  • django/db/models/base.py

     
    1212from django.dispatch import dispatcher
    1313from django.utils.datastructures import SortedDict
    1414from django.utils.functional import curry
     15from django.utils.file import file_move_safe
    1516from django.conf import settings
    1617from itertools import izip
    1718import types
    1819import sys
    1920import os
    2021
     22               
    2123class ModelBase(type):
    2224    "Metaclass for all models"
    2325    def __new__(cls, name, bases, attrs):
     
    361363    def _get_FIELD_size(self, field):
    362364        return os.path.getsize(self._get_FIELD_filename(field))
    363365
    364     def _save_FIELD_file(self, field, filename, raw_contents, save=True):
     366    def _save_FIELD_file(self, field, filename, raw_field, save=True):
    365367        directory = field.get_directory_name()
    366368        try: # Create the date-based directory if it doesn't exist.
    367369            os.makedirs(os.path.join(settings.MEDIA_ROOT, directory))
     
    383385        setattr(self, field.attname, filename)
    384386
    385387        full_filename = self._get_FIELD_filename(field)
    386         fp = open(full_filename, 'wb')
    387         fp.write(raw_contents)
    388         fp.close()
     388        if raw_field.has_key('tmpfilename'):
     389            raw_field['tmpfile'].close()
     390            file_move_safe(raw_field['tmpfilename'], full_filename)
     391        else:
     392            fp = open(full_filename, 'wb')
     393            fp.write(raw_field['content'])
     394            fp.close()
    389395
    390396        # Save the width and/or height, if applicable.
    391397        if isinstance(field, ImageField) and (field.width_field or field.height_field):
  • django/db/models/fields/__init__.py

     
    636636        setattr(cls, 'get_%s_filename' % self.name, curry(cls._get_FIELD_filename, field=self))
    637637        setattr(cls, 'get_%s_url' % self.name, curry(cls._get_FIELD_url, field=self))
    638638        setattr(cls, 'get_%s_size' % self.name, curry(cls._get_FIELD_size, field=self))
    639         setattr(cls, 'save_%s_file' % self.name, lambda instance, filename, raw_contents, save=True: instance._save_FIELD_file(self, filename, raw_contents, save))
     639        setattr(cls, 'save_%s_file' % self.name, lambda instance, filename, raw_field, save=True: instance._save_FIELD_file(self, filename, raw_field, save))
    640640        dispatcher.connect(self.delete_file, signal=signals.post_delete, sender=cls)
    641641
    642642    def delete_file(self, instance):
     
    659659        if new_data.get(upload_field_name, False):
    660660            func = getattr(new_object, 'save_%s_file' % self.name)
    661661            if rel:
    662                 func(new_data[upload_field_name][0]["filename"], new_data[upload_field_name][0]["content"], save)
     662                func(new_data[upload_field_name][0]["filename"], new_data[upload_field_name][0], save)
    663663            else:
    664                 func(new_data[upload_field_name]["filename"], new_data[upload_field_name]["content"], save)
     664                func(new_data[upload_field_name]["filename"], new_data[upload_field_name], save)
    665665
    666666    def get_directory_name(self):
    667667        return os.path.normpath(datetime.datetime.now().strftime(self.upload_to))
  • django/oldforms/__init__.py

     
    666666        self.validator_list = [self.isNonEmptyFile] + validator_list
    667667
    668668    def isNonEmptyFile(self, field_data, all_data):
    669         try:
    670             content = field_data['content']
    671         except TypeError:
     669        if field_data.has_key('_file_upload_error'):
     670            raise validators.CriticalValidationError, field_data['_file_upload_error']
     671        if not field_data.has_key('filename'):
    672672            raise validators.CriticalValidationError, gettext("No file was submitted. Check the encoding type on the form.")
    673         if not content:
     673        if not field_data['content-length']:
    674674            raise validators.CriticalValidationError, gettext("The submitted file is empty.")
    675675
    676676    def render(self, data):
    677677        return '<input type="file" id="%s" class="v%s" name="%s" />' % \
    678678            (self.get_id(), self.__class__.__name__, self.field_name)
    679679
     680    def prepare(self, new_data):
     681        if new_data.has_key('_file_upload_error'):
     682            # pretend we got something in the field to raise a validation error later
     683            new_data[self.field_name] = { '_file_upload_error': new_data['_file_upload_error'] }
     684
    680685    def html2python(data):
    681686        if data is None:
    682687            raise EmptyValue
  • django/core/handlers/wsgi.py

     
    7575        self.environ = environ
    7676        self.path = environ['PATH_INFO']
    7777        self.META = environ
     78        self.META['UPLOAD_PROGRESS_ID'] = self._get_file_progress_id()
    7879        self.method = environ['REQUEST_METHOD'].upper()
    7980
    8081    def __repr__(self):
     
    111112            if self.environ.get('CONTENT_TYPE', '').startswith('multipart'):
    112113                header_dict = dict([(k, v) for k, v in self.environ.items() if k.startswith('HTTP_')])
    113114                header_dict['Content-Type'] = self.environ.get('CONTENT_TYPE', '')
    114                 self._post, self._files = http.parse_file_upload(header_dict, self.raw_post_data)
     115                header_dict['Content-Length'] = self.environ.get('CONTENT_LENGTH', '')
     116                header_dict['X-Progress-ID'] = self.environ.get('HTTP_X_PROGRESS_ID', '')
     117                try:
     118                    self._post, self._files = http.parse_file_upload(header_dict, self.environ['wsgi.input'], self)
     119                except:
     120                    self._post, self._files = {}, {} # make sure we dont read the input stream again
     121                    raise
     122                self._raw_post_data = None # raw data is not available for streamed multipart messages
    115123            else:
    116124                self._post, self._files = http.QueryDict(self.raw_post_data), datastructures.MultiValueDict()
    117125        else:
     
    167175            buf.close()
    168176            return self._raw_post_data
    169177
     178    def _get_file_progress_id(self):
     179        """
     180        Returns the Progress ID of the request,
     181        usually provided if there is a file upload
     182        going on.
     183        Returns ``None`` if no progress ID is specified.
     184        """
     185        return self._get_file_progress_from_args(self.environ,
     186                                                 self.GET,
     187                                                 self._req.args)
     188
    170189    GET = property(_get_get, _set_get)
    171190    POST = property(_get_post, _set_post)
    172191    COOKIES = property(_get_cookies, _set_cookies)
  • django/core/handlers/base.py

     
    55
    66class BaseHandler(object):
    77    def __init__(self):
    8         self._request_middleware = self._view_middleware = self._response_middleware = self._exception_middleware = None
     8        self._upload_middleware = self._request_middleware = self._view_middleware = self._response_middleware = self._exception_middleware = None
    99
    1010    def load_middleware(self):
    1111        """
     
    1919        self._view_middleware = []
    2020        self._response_middleware = []
    2121        self._exception_middleware = []
     22        self._upload_middleware = []
    2223        for middleware_path in settings.MIDDLEWARE_CLASSES:
    2324            try:
    2425                dot = middleware_path.rindex('.')
     
    4748                self._response_middleware.insert(0, mw_instance.process_response)
    4849            if hasattr(mw_instance, 'process_exception'):
    4950                self._exception_middleware.insert(0, mw_instance.process_exception)
     51            if hasattr(mw_instance, 'process_upload'):
     52                self._upload_middleware.append(0, mw_instance.process_upload)
    5053
     54    def file_progress_descriptor(self, request):
     55        """
     56        Returns a descriptor that manages the file_progress
     57        """       
     58        for mw_call in self._upload_middleware:
     59            result = mw_call(http.MultiPartParserError)
     60            if result != None:
     61                return result
     62
     63        return http.DefaultFileProgressDescriptor(http.MultiPartParserError)
     64       
    5165    def get_response(self, request):
    5266        "Returns an HttpResponse object for the given HttpRequest"
    5367        from django.core import exceptions, urlresolvers
    5468        from django.core.mail import mail_admins
    5569        from django.conf import settings
    5670
     71        # Add file_progress descriptor
     72        request._file_progress = self.file_progress_descriptor(request)
     73
    5774        # Apply request middleware
    5875        for middleware_method in self._request_middleware:
    5976            response = middleware_method(request)
  • django/core/handlers/modpython.py

     
    4747    def _load_post_and_files(self):
    4848        "Populates self._post and self._files"
    4949        if 'content-type' in self._req.headers_in and self._req.headers_in['content-type'].startswith('multipart'):
    50             self._post, self._files = http.parse_file_upload(self._req.headers_in, self.raw_post_data)
     50            self._raw_post_data = None # raw data is not available for streamed multipart messages
     51            try:
     52                self._post, self._files = http.parse_file_upload(self._req.headers_in, self._req, self)
     53            except:
     54                self._post, self._files = {}, {} # make sure we dont read the input stream again
     55                raise
    5156        else:
    5257            self._post, self._files = http.QueryDict(self.raw_post_data), datastructures.MultiValueDict()
    5358
     
    9297                'AUTH_TYPE':         self._req.ap_auth_type,
    9398                'CONTENT_LENGTH':    self._req.clength, # This may be wrong
    9499                'CONTENT_TYPE':      self._req.content_type, # This may be wrong
    95                 'GATEWAY_INTERFACE': 'CGI/1.1',
    96                 'PATH_INFO':         self._req.path_info,
    97                 'PATH_TRANSLATED':   None, # Not supported
    98                 'QUERY_STRING':      self._req.args,
    99                 'REMOTE_ADDR':       self._req.connection.remote_ip,
    100                 'REMOTE_HOST':       None, # DNS lookups not supported
    101                 'REMOTE_IDENT':      self._req.connection.remote_logname,
    102                 'REMOTE_USER':       self._req.user,
    103                 'REQUEST_METHOD':    self._req.method,
    104                 'SCRIPT_NAME':       None, # Not supported
    105                 'SERVER_NAME':       self._req.server.server_hostname,
    106                 'SERVER_PORT':       self._req.server.port,
    107                 'SERVER_PROTOCOL':   self._req.protocol,
    108                 'SERVER_SOFTWARE':   'mod_python'
     100                'GATEWAY_INTERFACE':  'CGI/1.1',
     101                'PATH_INFO':          self._req.path_info,
     102                'PATH_TRANSLATED':    None, # Not supported
     103                'QUERY_STRING':       self._req.args,
     104                'REMOTE_ADDR':        self._req.connection.remote_ip,
     105                'REMOTE_HOST':        None, # DNS lookups not supported
     106                'REMOTE_IDENT':       self._req.connection.remote_logname,
     107                'REMOTE_USER':        self._req.user,
     108                'REQUEST_METHOD':     self._req.method,
     109                'SCRIPT_NAME':        None, # Not supported
     110                'SERVER_NAME':        self._req.server.server_hostname,
     111                'SERVER_PORT':        self._req.server.port,
     112                'SERVER_PROTOCOL':    self._req.protocol,
     113                'UPLOAD_PROGRESS_ID': self._get_file_progress_id(),
     114                'SERVER_SOFTWARE':    'mod_python'
    109115            }
    110116            for key, value in self._req.headers_in.items():
    111117                key = 'HTTP_' + key.upper().replace('-', '_')
     
    122128    def _get_method(self):
    123129        return self.META['REQUEST_METHOD'].upper()
    124130
     131    def _get_file_progress_id(self):
     132        """
     133        Returns the Progress ID of the request,
     134        usually provided if there is a file upload
     135        going on.
     136        Returns ``None`` if no progress ID is specified.
     137        """
     138        return self._get_file_progress_from_args(self._req.headers_in,
     139                                                 self.GET,
     140                                                 self._req.args)
     141
     142
    125143    GET = property(_get_get, _set_get)
    126144    POST = property(_get_post, _set_post)
    127145    COOKIES = property(_get_cookies, _set_cookies)
  • django/utils/file.py

     
     1import os
     2
     3try:
     4    import shutils
     5    file_move = shutils.move
     6except:
     7    file_move = os.rename
     8
     9def file_move_safe(old_file_name, new_file_name, chunk_size = 1024*64):
     10    """
     11    Moves a file from one location to another in the safest way possible.
     12   
     13    First, it tries using shutils.move, which is OS-dependent but doesn't
     14    break with change of filesystems. Then it tries os.rename, which will
     15    break if it encounters a change in filesystems. Lastly, it streams
     16    it manually from one file to another in python.
     17    """
     18   
     19    try:
     20        file_move(old_file_name, new_file_name)
     21        return
     22    except:
     23        pass
     24   
     25    new_file = open(new_file_name, 'wb')
     26    old_file = open(old_file_name, 'rb')
     27    current_chunk = None
     28   
     29    while current_chunk != '':
     30        current_chunk = old_file.read(chunk_size)
     31        new_file.write(current_chunk)
     32       
     33    new_file.close()
     34    old_file.close()
     35
     36    os.remove(old_file_name)
  • tests/modeltests/test_client/views.py

     
    4444
    4545    return HttpResponse(t.render(c))
    4646
     47def post_file_view(request):
     48    "A view that expects a multipart post and returns a file in the context"
     49    t = Template('File {{ file.filename }} received', name='POST Template')
     50    c = Context({'file': request.FILES['file_file']})
     51    return HttpResponse(t.render(c))
     52
    4753def redirect_view(request):
    4854    "A view that redirects all requests to the GET view"
    4955    return HttpResponseRedirect('/test_client/get_view/')
  • tests/modeltests/test_client/models.py

     
    7575        self.assertEqual(response.template.name, "Book template")
    7676        self.assertEqual(response.content, "Blink - Malcolm Gladwell")
    7777
     78    def test_post_file_view(self):
     79        "POST this python file to a view"
     80        import os, tempfile
     81        from django.conf import settings
     82        file = __file__.replace('.pyc', '.py')
     83        for upload_dir in [None, tempfile.gettempdir()]:
     84            settings.FILE_UPLOAD_DIR = upload_dir
     85            post_data = { 'name': file, 'file': open(file) }
     86            response = self.client.post('/test_client/post_file_view/', post_data)
     87            self.failUnless('models.py' in response.context['file']['filename'])
     88            self.failUnless(len(response.context['file']['content']) == os.path.getsize(file))
     89            if upload_dir:
     90                self.failUnless(response.context['file']['tmpfilename'])
     91
     92
    7893    def test_redirect(self):
    7994        "GET a URL that redirects elsewhere"
    8095        response = self.client.get('/test_client/redirect_view/')
  • tests/modeltests/test_client/urls.py

     
    44urlpatterns = patterns('',
    55    (r'^get_view/$', views.get_view),
    66    (r'^post_view/$', views.post_view),
     7    (r'^post_file_view/$', views.post_file_view),
    78    (r'^raw_post_view/$', views.raw_post_view),
    89    (r'^redirect_view/$', views.redirect_view),
    910    (r'^form_view/$', views.form_view),
  • docs/request_response.txt

     
    7272``FILES``
    7373    A dictionary-like object containing all uploaded files. Each key in
    7474    ``FILES`` is the ``name`` from the ``<input type="file" name="" />``. Each
    75     value in ``FILES`` is a standard Python dictionary with the following three
     75    value in ``FILES`` is a standard Python dictionary with the following four
    7676    keys:
    7777
    7878        * ``filename`` -- The name of the uploaded file, as a Python string.
    7979        * ``content-type`` -- The content type of the uploaded file.
    8080        * ``content`` -- The raw content of the uploaded file.
     81        * ``content-length`` -- The length of the content in bytes.
    8182
     83    If streaming file uploads are enabled two additional keys
     84    describing the uploaded file will be present:
     85
     86        * ``tmpfilename`` -- The filename for the temporary file.
     87        * ``tmpfile`` -- An open file object for the temporary file.
     88
     89    The temporary file will be removed when the request finishes.
     90
     91    Note that accessing ``content`` when streaming uploads are enabled
     92    will read the whole file into memory which may not be what you want.
     93
    8294    Note that ``FILES`` will only contain data if the request method was POST
    8395    and the ``<form>`` that posted to the request had
    8496    ``enctype="multipart/form-data"``. Otherwise, ``FILES`` will be a blank
  • docs/settings.txt

     
    438438
    439439.. _Testing Django Applications: ../testing/
    440440
     441FILE_UPLOAD_DIR
     442---------------
     443
     444Default: ``None``
     445
     446Path to a directory where temporary files should be written during
     447file uploads. Leaving this as ``None`` will disable streaming file uploads,
     448and cause all uploaded files to be stored (temporarily) in memory.
     449
    441450IGNORABLE_404_ENDS
    442451------------------
    443452
     
    781790
    782791.. _site framework docs: ../sites/
    783792
     793STREAMING_MIN_POST_SIZE
     794-----------------------
     795
     796Default: 524288 (``512*1024``)
     797
     798An integer specifying the minimum number of bytes that has to be
     799received (in a POST) for file upload streaming to take place. Any
     800request smaller than this will be handled in memory.
     801Note: ``FILE_UPLOAD_DIR`` has to be defined to enable streaming.
     802
    784803TEMPLATE_CONTEXT_PROCESSORS
    785804---------------------------
    786805
  • docs/forms.txt

     
    475475   new_data = request.POST.copy()
    476476   new_data.update(request.FILES)
    477477
     478Streaming file uploads.
     479-----------------------
     480
     481File uploads will be read into memory by default. This works fine for
     482small to medium sized uploads (from 1MB to 100MB depending on your
     483setup and usage). If you want to support larger uploads you can enable
     484upload streaming where only a small part of the file will be in memory
     485at any time. To do this you need to specify the ``FILE_UPLOAD_DIR``
     486setting (see the settings_ document for more details).
     487
     488See `request object`_ for more details about ``request.FILES`` objects
     489with streaming file uploads enabled.
     490
    478491Validators
    479492==========
    480493
     
    697710.. _`generic views`: ../generic_views/
    698711.. _`models API`: ../model-api/
    699712.. _settings: ../settings/
     713.. _request object: ../request_response/#httprequest-objects
Back to Top