Ticket #2070: 5343_1_streaming_file_upload.diff

File 5343_1_streaming_file_upload.diff, 38.4 KB (added by Michael Axiak <axiak@…>, 8 years ago)

Newer, cleaner version of file upload script

  • django/http/multipartparser.py

     
     1"""
     2MultiPart parsing for file uploads.
     3If both a progress id is sent (either through ``X-Progress-ID``
     4header or ``progress_id`` GET) and ``FILE_UPLOAD_DIR`` is set
     5in the settings, then the file progress will be tracked using
     6``request.file_progress``.
     7
     8To use this feature, consider creating a middleware with an appropriate
     9``process_request``::
     10
     11    class FileProgressTrack(object):
     12        def __get__(self, request, HttpRequest):
     13            progress_id = request.META['UPLOAD_PROGRESS_ID']
     14            status = # get progress from progress_id here
     15
     16            return status
     17
     18        def __set__(self, request, new_value):
     19            progress_id = request.META['UPLOAD_PROGRESS_ID']
     20
     21            # set the progress using progress_id here.
     22
     23    # example middleware
     24    class FileProgressExample(object):
     25        def process_request(self, request):
     26            request.__class__.file_progress = FileProgressTrack()
     27
     28
     29
     30"""
     31
     32__all__ = ['MultiPartParserError','MultiPartParser']
     33
     34
     35from django.utils.datastructures import MultiValueDict
     36import os
     37
     38try:
     39    from cStringIO import StringIO
     40except ImportError:
     41    from StringIO import StringIO
     42
     43
     44class MultiPartParserError(Exception):
     45    def __init__(self, message):
     46        self.message = message
     47    def __str__(self):
     48        return repr(self.message)
     49
     50class MultiPartParser(object):
     51    """
     52    A rfc2388 multipart/form-data parser.
     53   
     54    parse() reads the input stream in chunk_size chunks and returns a
     55    tuple of (POST MultiValueDict, FILES MultiValueDict). If
     56    file_upload_dir is defined files will be streamed to temporary
     57    files in the specified directory.
     58
     59    The FILES dictionary will have 'filename', 'content-type',
     60    'content' and 'content-length' entries. For streamed files it will
     61    also have 'tmpfilename' and 'tmpfile'. The 'content' entry will
     62    only be read from disk when referenced for streamed files.
     63
     64    If the X-Progress-ID is sent (in one of many formats), then
     65    object.file_progress will be given a dictionary of the progress.
     66    """
     67    def __init__(self, headers, input, request, file_upload_dir=None, streaming_min_post_size=None, chunk_size=1024*64):
     68        try:
     69            content_length = int(headers['Content-Length'])
     70        except:
     71            raise MultiPartParserError('Invalid Content-Length: %s' % headers.get('Content-Length'))
     72
     73        content_type = headers.get('Content-Type')
     74
     75        if not content_type or not content_type.startswith('multipart/'):
     76            raise MultiPartParserError('Invalid Content-Type: %s' % content_type)
     77           
     78        ctype, opts = self.parse_header(content_type)
     79        boundary = opts.get('boundary')
     80        from cgi import valid_boundary
     81        if not boundary or not valid_boundary(boundary):
     82            raise MultiPartParserError('Invalid boundary in multipart form: %s' % boundary)
     83
     84        progress_id = request.META['UPLOAD_PROGRESS_ID']
     85
     86        self._track_progress = file_upload_dir and progress_id # whether or not to track progress
     87        self._boundary = '--' + boundary
     88        self._input = input
     89        self._size = content_length
     90        self._received = 0
     91        self._file_upload_dir = file_upload_dir
     92        self._chunk_size = chunk_size
     93        self._state = 'PREAMBLE'
     94        self._partial = ''
     95        self._post = MultiValueDict()
     96        self._files = MultiValueDict()
     97        self._request = request
     98
     99        if streaming_min_post_size is not None and content_length < streaming_min_post_size:
     100            self._file_upload_dir = None # disable file streaming for small request
     101        elif self._track_progress:
     102            request.file_progress = {'state': 'starting'}
     103
     104        try:
     105            # Use mx fast string search if available.
     106            from mx.TextTools import FS
     107            self._fs = FS(self._boundary)
     108        except ImportError:
     109            self._fs = None
     110
     111    def parse(self):
     112        try:
     113            self._parse()
     114        finally:
     115            if self._track_progress:
     116                self._request.file_progress = {'state': 'done'}
     117        return self._post, self._files
     118
     119    def _parse(self):
     120        size = self._size
     121
     122        try:
     123            while size > 0:
     124                n = self._read(self._input, min(self._chunk_size, size))
     125                if not n:
     126                    break
     127                size -= n
     128        except:
     129            # consume any remaining data so we dont generate a "Connection Reset" error
     130            size = self._size - self._received
     131            while size > 0:
     132                data = self._input.read(min(self._chunk_size, size))
     133                size -= len(data)
     134            raise
     135
     136    def _find_boundary(self, data, start, stop):
     137        """
     138        Find the next boundary and return the end of current part
     139        and start of next part.
     140        """
     141        if self._fs:
     142            boundary = self._fs.find(data, start, stop)
     143        else:
     144            boundary = data.find(self._boundary, start, stop)
     145        if boundary >= 0:
     146            end = boundary
     147            next = boundary + len(self._boundary)
     148
     149            # backup over CRLF
     150            if end > 0 and data[end-1] == '\n': end -= 1
     151            if end > 0 and data[end-1] == '\r': end -= 1
     152            # skip over --CRLF
     153            if next < stop and data[next] == '-': next += 1
     154            if next < stop and data[next] == '-': next += 1
     155            if next < stop and data[next] == '\r': next += 1
     156            if next < stop and data[next] == '\n': next += 1
     157
     158            return True, end, next
     159        else:
     160            return False, stop, stop
     161
     162    class TemporaryFile(object):
     163        "A temporary file that tries to delete itself when garbage collected."
     164        def __init__(self, dir):
     165            import tempfile
     166            (fd, name) = tempfile.mkstemp(suffix='.upload', dir=dir)
     167            self.file = os.fdopen(fd, 'w+b')
     168            self.name = name
     169
     170        def __getattr__(self, name):
     171            a = getattr(self.__dict__['file'], name)
     172            if type(a) != type(0):
     173                setattr(self, name, a)
     174            return a
     175
     176        def __del__(self):
     177            try:
     178                os.unlink(self.name)
     179            except OSError:
     180                pass
     181
     182    class LazyContent(dict):
     183        """
     184        A lazy FILES dictionary entry that reads the contents from
     185        tmpfile only when referenced.
     186        """
     187        def __init__(self, data):
     188            dict.__init__(self, data)
     189
     190        def __getitem__(self, key):
     191            if key == 'content' and not self.has_key(key):
     192                self['tmpfile'].seek(0)
     193                self['content'] = self['tmpfile'].read()
     194            return dict.__getitem__(self, key)
     195
     196    def _read(self, input, size):
     197        data = input.read(size)
     198
     199        if not data:
     200            return 0
     201
     202        read_size = len(data)
     203        self._received += read_size
     204
     205        if self._partial:
     206            data = self._partial + data
     207
     208        start = 0
     209        stop = len(data)
     210
     211        while start < stop:
     212            boundary, end, next = self._find_boundary(data, start, stop)
     213
     214            if not boundary and read_size:
     215                # make sure we dont treat a partial boundary (and its separators) as data
     216                stop -= len(self._boundary) + 16
     217                end = next = stop
     218                if end <= start:
     219                    break # need more data
     220
     221            if self._state == 'PREAMBLE':
     222                # Preamble, just ignore it
     223                self._state = 'HEADER'
     224
     225            elif self._state == 'HEADER':
     226                # Beginning of header, look for end of header and parse it if found.
     227
     228                header_end = data.find('\r\n\r\n', start, stop)
     229                if header_end == -1:
     230                    break # need more data
     231
     232                header = data[start:header_end]
     233
     234                self._fieldname = None
     235                self._filename = None
     236                self._content_type = None
     237
     238                for line in header.split('\r\n'):
     239                    ctype, opts = self.parse_header(line)
     240                    if ctype == 'content-disposition: form-data':
     241                        self._fieldname = opts.get('name')
     242                        self._filename = opts.get('filename')
     243                    elif ctype.startswith('content-type: '):
     244                        self._content_type = ctype[14:]
     245
     246                if self._filename is not None:
     247                    # cleanup filename from IE full paths:
     248                    self._filename = self._filename[self._filename.rfind("\\")+1:].strip()
     249
     250                    if self._filename: # ignore files without filenames
     251                        if self._file_upload_dir:
     252#                            assert False, 'hmm'
     253                           
     254                            try:
     255                                self._file = self.TemporaryFile(dir=self._file_upload_dir)
     256                            except OSError, IOError:
     257                                raise MultiPartParserError("Failed to create temporary file. Error was %s" % e)
     258                        else:
     259                            self._file = StringIO()
     260                    else:
     261                        self._file = None
     262                    self._filesize = 0
     263                    self._state = 'FILE'
     264                else:
     265                    self._field = StringIO()
     266                    self._state = 'FIELD'
     267                next = header_end + 4
     268
     269            elif self._state == 'FIELD':
     270                # In a field, collect data until a boundary is found.
     271
     272                self._field.write(data[start:end])
     273                if boundary:
     274                    if self._fieldname:
     275                        self._post.appendlist(self._fieldname, self._field.getvalue())
     276                    self._field.close()
     277                    self._state = 'HEADER'
     278
     279            elif self._state == 'FILE':
     280                # In a file, collect data until a boundary is found.
     281
     282                if self._file:
     283                    try:
     284                        self._file.write(data[start:end])
     285                    except IOError, e:
     286                        raise MultiPartParserError("Failed to write to temporary file.")
     287                    self._filesize += end-start
     288
     289                    if self._track_progress:
     290                        self._request.file_progress = {'received': self._received,
     291                                                       'size':     self._size,
     292                                                       'state':    'uploading'}
     293
     294                if boundary:
     295                    if self._file:
     296                        if self._file_upload_dir:
     297                            self._file.seek(0)
     298                            file = self.LazyContent({
     299                                'filename': self._filename,
     300                                'content-type':  self._content_type,
     301                                # 'content': is read on demand
     302                                'content-length': self._filesize,
     303                                'tmpfilename': self._file.name,
     304                                'tmpfile': self._file
     305                            })
     306                        else:
     307                            file = {
     308                                'filename': self._filename,
     309                                'content-type':  self._content_type,
     310                                'content': self._file.getvalue(),
     311                                'content-length': self._filesize
     312                            }
     313                            self._file.close()
     314
     315                        self._files.appendlist(self._fieldname, file)
     316
     317                    self._state = 'HEADER'
     318
     319            start = next
     320
     321        self._partial = data[start:]
     322
     323        return read_size
     324
     325    def parse_header(self, line):
     326        from cgi import parse_header
     327        return parse_header(line)
  • django/http/__init__.py

     
    1 import os
     1import os, pickle
    22from Cookie import SimpleCookie
    33from pprint import pformat
    44from urllib import urlencode, quote
    55from django.utils.datastructures import MultiValueDict
     6from django.http.multipartparser import MultiPartParser, MultiPartParserError
     7import re
    68
     9upload_id_re = re.compile(r'^[a-fA-F0-9]{32}$') # file progress id Regular expression
     10
    711RESERVED_CHARS="!*'();:@&=+$,/?%#[]"
    812
    913try:
     
    1721
    1822class HttpRequest(object):
    1923    "A basic HTTP request"
     24
    2025    def __init__(self):
    2126        self.GET, self.POST, self.COOKIES, self.META, self.FILES = {}, {}, {}, {}, {}
    2227        self.path = ''
     
    4247    def is_secure(self):
    4348        return os.environ.get("HTTPS") == "on"
    4449
    45 def parse_file_upload(header_dict, post_data):
    46     "Returns a tuple of (POST MultiValueDict, FILES MultiValueDict)"
    47     import email, email.Message
    48     from cgi import parse_header
    49     raw_message = '\r\n'.join(['%s:%s' % pair for pair in header_dict.items()])
    50     raw_message += '\r\n\r\n' + post_data
    51     msg = email.message_from_string(raw_message)
    52     POST = MultiValueDict()
    53     FILES = MultiValueDict()
    54     for submessage in msg.get_payload():
    55         if submessage and isinstance(submessage, email.Message.Message):
    56             name_dict = parse_header(submessage['Content-Disposition'])[1]
    57             # name_dict is something like {'name': 'file', 'filename': 'test.txt'} for file uploads
    58             # or {'name': 'blah'} for POST fields
    59             # We assume all uploaded files have a 'filename' set.
    60             if 'filename' in name_dict:
    61                 assert type([]) != type(submessage.get_payload()), "Nested MIME messages are not supported"
    62                 if not name_dict['filename'].strip():
    63                     continue
    64                 # IE submits the full path, so trim everything but the basename.
    65                 # (We can't use os.path.basename because it expects Linux paths.)
    66                 filename = name_dict['filename'][name_dict['filename'].rfind("\\")+1:]
    67                 FILES.appendlist(name_dict['name'], {
    68                     'filename': filename,
    69                     'content-type': 'Content-Type' in submessage and submessage['Content-Type'] or None,
    70                     'content': submessage.get_payload(),
    71                 })
    72             else:
    73                 POST.appendlist(name_dict['name'], submessage.get_payload())
    74     return POST, FILES
     50    def _get_file_progress(self):
     51        return {}
    7552
     53    def _set_file_progress(self,value):
     54        pass
     55
     56    def _del_file_progress(self):
     57        pass
     58
     59    file_progress = property(_get_file_progress,
     60                             _set_file_progress,
     61                             _del_file_progress)
     62
     63    def _get_file_progress_from_args(self, headers, get, querystring):
     64        """
     65        This parses the request for a file progress_id value.
     66        Note that there are two distinct ways of getting the progress
     67        ID -- header and GET. One is used primarily to attach via JavaScript
     68        to the end of an HTML form action while the other is used for AJAX
     69        communication.
     70
     71        All progress IDs must be valid 32-digit hexadecimal numbers.
     72        """
     73        if 'X-Progress-ID' in headers:
     74            progress_id = headers['X-Upload-ID']
     75        elif 'progress_id' in get:
     76            progress_id = get['progress_id']
     77        else:
     78            return None
     79
     80        if not self.upload_id_re.match(progress_id):
     81            return None
     82
     83        return progress_id
     84
     85def parse_file_upload(headers, input, request):
     86    from django.conf import settings
     87
     88    # Only stream files to disk if FILE_STREAMING_DIR is set
     89    file_upload_dir = settings.FILE_UPLOAD_DIR
     90    streaming_min_post_size = settings.STREAMING_MIN_POST_SIZE
     91
     92    try:
     93        parser = MultiPartParser(headers, input, request, file_upload_dir, streaming_min_post_size)
     94        return parser.parse()
     95    except MultiPartParserError, e:
     96        return MultiValueDict({ '_file_upload_error': [e.message] }), {}
     97
     98
    7699class QueryDict(MultiValueDict):
    77100    """A specialized MultiValueDict that takes a query string when initialized.
    78101    This is immutable unless you create a copy of it."""
  • django/oldforms/__init__.py

     
    666666        self.validator_list = [self.isNonEmptyFile] + validator_list
    667667
    668668    def isNonEmptyFile(self, field_data, all_data):
    669         try:
    670             content = field_data['content']
    671         except TypeError:
     669        if field_data.has_key('_file_upload_error'):
     670            raise validators.CriticalValidationError, field_data['_file_upload_error']
     671        if not field_data.has_key('filename'):
    672672            raise validators.CriticalValidationError, gettext("No file was submitted. Check the encoding type on the form.")
    673         if not content:
     673        if not field_data['content-length']:
    674674            raise validators.CriticalValidationError, gettext("The submitted file is empty.")
    675675
    676676    def render(self, data):
    677677        return '<input type="file" id="%s" class="v%s" name="%s" />' % \
    678678            (self.get_id(), self.__class__.__name__, self.field_name)
    679679
     680    def prepare(self, new_data):
     681        if new_data.has_key('_file_upload_error'):
     682            # pretend we got something in the field to raise a validation error later
     683            new_data[self.field_name] = { '_file_upload_error': new_data['_file_upload_error'] }
     684
    680685    def html2python(data):
    681686        if data is None:
    682687            raise EmptyValue
  • django/db/models/base.py

     
    1212from django.dispatch import dispatcher
    1313from django.utils.datastructures import SortedDict
    1414from django.utils.functional import curry
     15from django.utils.file import file_move_safe
    1516from django.conf import settings
    1617from itertools import izip
    1718import types
    1819import sys
    1920import os
    2021
     22               
    2123class ModelBase(type):
    2224    "Metaclass for all models"
    2325    def __new__(cls, name, bases, attrs):
     
    361363    def _get_FIELD_size(self, field):
    362364        return os.path.getsize(self._get_FIELD_filename(field))
    363365
    364     def _save_FIELD_file(self, field, filename, raw_contents, save=True):
     366    def _save_FIELD_file(self, field, filename, raw_field, save=True):
    365367        directory = field.get_directory_name()
    366368        try: # Create the date-based directory if it doesn't exist.
    367369            os.makedirs(os.path.join(settings.MEDIA_ROOT, directory))
    368370        except OSError: # Directory probably already exists.
    369371            pass
     372
     373        if filename is None:
     374            filename = raw_field['filename']
     375
    370376        filename = field.get_filename(filename)
    371377
    372378        # If the filename already exists, keep adding an underscore to the name of
     
    383389        setattr(self, field.attname, filename)
    384390
    385391        full_filename = self._get_FIELD_filename(field)
    386         fp = open(full_filename, 'wb')
    387         fp.write(raw_contents)
    388         fp.close()
     392        if raw_field.has_key('tmpfilename'):
     393            raw_field['tmpfile'].close()
     394            file_move_safe(raw_field['tmpfilename'], full_filename)
     395        else:
     396            from django.utils import file_locks
     397            fp = open(full_filename, 'wb')
     398            # exclusive lock
     399            file_locks.lock(fp, file_locks.LOCK_EX)
     400            fp.write(raw_field['content'])
     401            fp.close()
    389402
    390403        # Save the width and/or height, if applicable.
    391404        if isinstance(field, ImageField) and (field.width_field or field.height_field):
  • django/db/models/fields/__init__.py

     
    701701        setattr(cls, 'get_%s_filename' % self.name, curry(cls._get_FIELD_filename, field=self))
    702702        setattr(cls, 'get_%s_url' % self.name, curry(cls._get_FIELD_url, field=self))
    703703        setattr(cls, 'get_%s_size' % self.name, curry(cls._get_FIELD_size, field=self))
    704         setattr(cls, 'save_%s_file' % self.name, lambda instance, filename, raw_contents, save=True: instance._save_FIELD_file(self, filename, raw_contents, save))
     704        setattr(cls, 'save_%s_file' % self.name, lambda instance, filename, raw_field, save=True: instance._save_FIELD_file(self, filename, raw_field, save))
     705        setattr(cls, 'move_%s_file' % self.name, lambda instance, raw_field, save=True: instance._save_FIELD_file(self, None, raw_field, save))       
    705706        dispatcher.connect(self.delete_file, signal=signals.post_delete, sender=cls)
    706707
    707708    def delete_file(self, instance):
     
    724725        if new_data.get(upload_field_name, False):
    725726            func = getattr(new_object, 'save_%s_file' % self.name)
    726727            if rel:
    727                 func(new_data[upload_field_name][0]["filename"], new_data[upload_field_name][0]["content"], save)
     728                func(new_data[upload_field_name][0]["filename"], new_data[upload_field_name][0], save)
    728729            else:
    729                 func(new_data[upload_field_name]["filename"], new_data[upload_field_name]["content"], save)
     730                func(new_data[upload_field_name]["filename"], new_data[upload_field_name], save)
    730731
    731732    def get_directory_name(self):
    732733        return os.path.normpath(datetime.datetime.now().strftime(self.upload_to))
  • django/conf/global_settings.py

     
    242242# isExistingURL validator.
    243243URL_VALIDATOR_USER_AGENT = "Django/0.96pre (http://www.djangoproject.com)"
    244244
     245# The directory to place streamed file uploads. The web server needs write
     246# permissions on this directory.
     247# If this is None, streaming uploads are disabled.
     248FILE_UPLOAD_DIR = None
     249
     250# The minimum size of a POST before file uploads are streamed to disk.
     251# Any less than this number, and the file is uploaded to memory.
     252# Size is in bytes.
     253STREAMING_MIN_POST_SIZE = 512 * (2**10)
     254
    245255##############
    246256# MIDDLEWARE #
    247257##############
  • django/core/handlers/wsgi.py

     
    7575        self.environ = environ
    7676        self.path = environ['PATH_INFO']
    7777        self.META = environ
     78        self.META['UPLOAD_PROGRESS_ID'] = self._get_file_progress_id()
    7879        self.method = environ['REQUEST_METHOD'].upper()
    7980
    8081    def __repr__(self):
     
    111112            if self.environ.get('CONTENT_TYPE', '').startswith('multipart'):
    112113                header_dict = dict([(k, v) for k, v in self.environ.items() if k.startswith('HTTP_')])
    113114                header_dict['Content-Type'] = self.environ.get('CONTENT_TYPE', '')
    114                 self._post, self._files = http.parse_file_upload(header_dict, self.raw_post_data)
     115                header_dict['Content-Length'] = self.environ.get('CONTENT_LENGTH', '')
     116                header_dict['X-Progress-ID'] = self.environ.get('HTTP_X_PROGRESS_ID', '')
     117                try:
     118                    self._post, self._files = http.parse_file_upload(header_dict, self.environ['wsgi.input'], self)
     119                except:
     120                    self._post, self._files = {}, {} # make sure we dont read the input stream again
     121                    raise
     122                self._raw_post_data = None # raw data is not available for streamed multipart messages
    115123            else:
    116124                self._post, self._files = http.QueryDict(self.raw_post_data), datastructures.MultiValueDict()
    117125        else:
     
    167175            buf.close()
    168176            return self._raw_post_data
    169177
     178    def _get_file_progress_id(self):
     179        """
     180        Returns the Progress ID of the request,
     181        usually provided if there is a file upload
     182        going on.
     183        Returns ``None`` if no progress ID is specified.
     184        """
     185        return self._get_file_progress_from_args(self.environ,
     186                                                 self.GET,
     187                                                 self.environ.get('QUERY_STRING', ''))
     188
    170189    GET = property(_get_get, _set_get)
    171190    POST = property(_get_post, _set_post)
    172191    COOKIES = property(_get_cookies, _set_cookies)
  • django/core/handlers/base.py

     
    11from django.core import signals
    22from django.dispatch import dispatcher
    33from django import http
    4 import sys
     4import sys, re
    55
    66class BaseHandler(object):
    77    def __init__(self):
     
    129129        "Helper function to return the traceback as a string"
    130130        import traceback
    131131        return '\n'.join(traceback.format_exception(*(exc_info or sys.exc_info())))
     132
     133
  • django/core/handlers/modpython.py

     
    4747    def _load_post_and_files(self):
    4848        "Populates self._post and self._files"
    4949        if 'content-type' in self._req.headers_in and self._req.headers_in['content-type'].startswith('multipart'):
    50             self._post, self._files = http.parse_file_upload(self._req.headers_in, self.raw_post_data)
     50            self._raw_post_data = None # raw data is not available for streamed multipart messages
     51            try:
     52                self._post, self._files = http.parse_file_upload(self._req.headers_in, self._req, self)
     53            except:
     54                self._post, self._files = {}, {} # make sure we dont read the input stream again
     55                raise
    5156        else:
    5257            self._post, self._files = http.QueryDict(self.raw_post_data), datastructures.MultiValueDict()
    5358
     
    9297                'AUTH_TYPE':         self._req.ap_auth_type,
    9398                'CONTENT_LENGTH':    self._req.clength, # This may be wrong
    9499                'CONTENT_TYPE':      self._req.content_type, # This may be wrong
    95                 'GATEWAY_INTERFACE': 'CGI/1.1',
    96                 'PATH_INFO':         self._req.path_info,
    97                 'PATH_TRANSLATED':   None, # Not supported
    98                 'QUERY_STRING':      self._req.args,
    99                 'REMOTE_ADDR':       self._req.connection.remote_ip,
    100                 'REMOTE_HOST':       None, # DNS lookups not supported
    101                 'REMOTE_IDENT':      self._req.connection.remote_logname,
    102                 'REMOTE_USER':       self._req.user,
    103                 'REQUEST_METHOD':    self._req.method,
    104                 'SCRIPT_NAME':       None, # Not supported
    105                 'SERVER_NAME':       self._req.server.server_hostname,
    106                 'SERVER_PORT':       self._req.server.port,
    107                 'SERVER_PROTOCOL':   self._req.protocol,
    108                 'SERVER_SOFTWARE':   'mod_python'
     100                'GATEWAY_INTERFACE':  'CGI/1.1',
     101                'PATH_INFO':          self._req.path_info,
     102                'PATH_TRANSLATED':    None, # Not supported
     103                'QUERY_STRING':       self._req.args,
     104                'REMOTE_ADDR':        self._req.connection.remote_ip,
     105                'REMOTE_HOST':        None, # DNS lookups not supported
     106                'REMOTE_IDENT':       self._req.connection.remote_logname,
     107                'REMOTE_USER':        self._req.user,
     108                'REQUEST_METHOD':     self._req.method,
     109                'SCRIPT_NAME':        None, # Not supported
     110                'SERVER_NAME':        self._req.server.server_hostname,
     111                'SERVER_PORT':        self._req.server.port,
     112                'SERVER_PROTOCOL':    self._req.protocol,
     113                'UPLOAD_PROGRESS_ID': self._get_file_progress_id(),
     114                'SERVER_SOFTWARE':    'mod_python'
    109115            }
    110116            for key, value in self._req.headers_in.items():
    111117                key = 'HTTP_' + key.upper().replace('-', '_')
     
    122128    def _get_method(self):
    123129        return self.META['REQUEST_METHOD'].upper()
    124130
     131    def _get_file_progress_id(self):
     132        """
     133        Returns the Progress ID of the request,
     134        usually provided if there is a file upload
     135        going on.
     136        Returns ``None`` if no progress ID is specified.
     137        """
     138        return self._get_file_progress_from_args(self._req.headers_in,
     139                                                 self.GET,
     140                                                 self._req.args)
     141
    125142    GET = property(_get_get, _set_get)
    126143    POST = property(_get_post, _set_post)
    127144    COOKIES = property(_get_cookies, _set_cookies)
  • django/utils/file_locks.py

     
     1"""
     2Locking portability by Jonathan Feignberg <jdf@pobox.com> in python cookbook
     3
     4Example Usage::
     5
     6    from django.utils import file_locks
     7
     8    f = open('./file', 'wb')
     9
     10    file_locks.lock(f, file_locks.LOCK_EX)
     11    f.write('Django')
     12    f.close()
     13"""
     14
     15
     16import os
     17
     18__all__ = ['LOCK_EX','LOCK_SH','LOCK_NB','lock','unlock']
     19
     20if os.name == 'nt':
     21        import win32con
     22        import win32file
     23        import pywintypes
     24        LOCK_EX = win32con.LOCKFILE_EXCLUSIVE_LOCK
     25        LOCK_SH = 0
     26        LOCK_NB = win32con.LOCKFILE_FAIL_IMMEDIATELY
     27        __overlapped = pywintypes.OVERLAPPED()
     28elif os.name == 'posix':
     29        import fcntl
     30        LOCK_EX = fcntl.LOCK_EX
     31        LOCK_SH = fcntl.LOCK_SH
     32        LOCK_NB = fcntl.LOCK_NB
     33else:
     34        raise RuntimeError("Locking only defined for nt and posix platforms")
     35
     36if os.name == 'nt':
     37        def lock(file, flags):
     38                hfile = win32file._get_osfhandle(file.fileno())
     39                win32file.LockFileEx(hfile, flags, 0, -0x10000, __overlapped)
     40
     41        def unlock(file):
     42                hfile = win32file._get_osfhandle(file.fileno())
     43                win32file.UnlockFileEx(hfile, 0, -0x10000, __overlapped)
     44
     45elif os.name =='posix':
     46        def lock(file, flags):
     47                fcntl.flock(file.fileno(), flags)
     48
     49        def unlock(file):
     50                fcntl.flock(file.fileno(), fcntl.LOCK_UN)
  • django/utils/file.py

     
     1import os
     2
     3__all__ = ['file_move_safe']
     4
     5try:
     6    import shutil
     7    file_move = shutil.move
     8except ImportError:
     9    file_move = os.rename
     10
     11def file_move_safe(old_file_name, new_file_name, chunk_size = 1024*64, allow_overwrite=False):
     12    """
     13    Moves a file from one location to another in the safest way possible.
     14   
     15    First, it tries using shutils.move, which is OS-dependent but doesn't
     16    break with change of filesystems. Then it tries os.rename, which will
     17    break if it encounters a change in filesystems. Lastly, it streams
     18    it manually from one file to another in python.
     19
     20    Without ``allow_overwrite``, if the destination file exists, the
     21    file will raise an IOError.
     22    """
     23
     24    from django.utils import file_locks
     25
     26    if old_file_name == new_file_name:
     27        # No file moving takes place.
     28        return
     29
     30    if not allow_overwrite and os.path.exists(new_file_name):
     31        raise IOError, "Django does not allow overwriting files."
     32
     33    try:
     34        file_move(old_file_name, new_file_name)
     35        return
     36    except OSError: # moving to another filesystem
     37        pass
     38
     39    new_file = open(new_file_name, 'wb')
     40    # exclusive lock
     41    file_locks.lock(new_file, file_locks.LOCK_EX)
     42    old_file = open(old_file_name, 'rb')
     43    current_chunk = None
     44
     45    while current_chunk != '':
     46        current_chunk = old_file.read(chunk_size)
     47        new_file.write(current_chunk)
     48
     49    new_file.close()
     50    old_file.close()
     51
     52    os.remove(old_file_name)
     53
  • tests/modeltests/test_client/views.py

     
    4646
    4747    return HttpResponse(t.render(c))
    4848
     49def post_file_view(request):
     50    "A view that expects a multipart post and returns a file in the context"
     51    t = Template('File {{ file.filename }} received', name='POST Template')
     52    c = Context({'file': request.FILES['file_file']})
     53    return HttpResponse(t.render(c))
     54
    4955def redirect_view(request):
    5056    "A view that redirects all requests to the GET view"
    5157    return HttpResponseRedirect('/test_client/get_view/')
  • tests/modeltests/test_client/models.py

     
    33
    44The test client is a class that can act like a simple
    55browser for testing purposes.
    6  
     6
    77It allows the user to compose GET and POST requests, and
    88obtain the response that the server gave to those requests.
    99The server Response objects are annotated with the details
     
    7676        self.assertEqual(response.template.name, "Book template")
    7777        self.assertEqual(response.content, "Blink - Malcolm Gladwell")
    7878
     79    def test_post_file_view(self):
     80        "POST this python file to a view"
     81        import os, tempfile
     82        from django.conf import settings
     83        file = __file__.replace('.pyc', '.py')
     84        for upload_dir in [None, tempfile.gettempdir()]:
     85            settings.FILE_UPLOAD_DIR = upload_dir
     86            post_data = { 'name': file, 'file': open(file) }
     87            response = self.client.post('/test_client/post_file_view/', post_data)
     88            self.failUnless('models.py' in response.context['file']['filename'])
     89            self.failUnless(len(response.context['file']['content']) == os.path.getsize(file))
     90            if upload_dir:
     91                self.failUnless(response.context['file']['tmpfilename'])
     92
    7993    def test_redirect(self):
    8094        "GET a URL that redirects elsewhere"
    8195        response = self.client.get('/test_client/redirect_view/')
  • tests/modeltests/test_client/urls.py

     
    55urlpatterns = patterns('',
    66    (r'^get_view/$', views.get_view),
    77    (r'^post_view/$', views.post_view),
     8    (r'^post_file_view/$', views.post_file_view),
    89    (r'^raw_post_view/$', views.raw_post_view),
    910    (r'^redirect_view/$', views.redirect_view),
    1011    (r'^permanent_redirect_view/$', redirect_to, { 'url': '/test_client/get_view/' }),
  • docs/request_response.txt

     
    7272``FILES``
    7373    A dictionary-like object containing all uploaded files. Each key in
    7474    ``FILES`` is the ``name`` from the ``<input type="file" name="" />``. Each
    75     value in ``FILES`` is a standard Python dictionary with the following three
     75    value in ``FILES`` is a standard Python dictionary with the following four
    7676    keys:
    7777
    7878        * ``filename`` -- The name of the uploaded file, as a Python string.
    7979        * ``content-type`` -- The content type of the uploaded file.
    8080        * ``content`` -- The raw content of the uploaded file.
     81        * ``content-length`` -- The length of the content in bytes.
    8182
     83    If streaming file uploads are enabled two additional keys
     84    describing the uploaded file will be present:
     85
     86        * ``tmpfilename`` -- The filename for the temporary file.
     87        * ``tmpfile`` -- An open file object for the temporary file.
     88
     89    The temporary file will be removed when the request finishes.
     90
     91    Note that accessing ``content`` when streaming uploads are enabled
     92    will read the whole file into memory which may not be what you want.
     93
    8294    Note that ``FILES`` will only contain data if the request method was POST
    8395    and the ``<form>`` that posted to the request had
    8496    ``enctype="multipart/form-data"``. Otherwise, ``FILES`` will be a blank
  • docs/settings.txt

     
    448448
    449449.. _Testing Django Applications: ../testing/
    450450
     451FILE_UPLOAD_DIR
     452---------------
     453
     454Default: ``None``
     455
     456Path to a directory where temporary files should be written during
     457file uploads. Leaving this as ``None`` will disable streaming file uploads,
     458and cause all uploaded files to be stored (temporarily) in memory.
     459
    451460IGNORABLE_404_ENDS
    452461------------------
    453462
     
    764773
    765774.. _site framework docs: ../sites/
    766775
     776STREAMING_MIN_POST_SIZE
     777-----------------------
     778
     779Default: 524288 (``512*1024``)
     780
     781An integer specifying the minimum number of bytes that has to be
     782received (in a POST) for file upload streaming to take place. Any
     783request smaller than this will be handled in memory.
     784Note: ``FILE_UPLOAD_DIR`` has to be defined to enable streaming.
     785
    767786TEMPLATE_CONTEXT_PROCESSORS
    768787---------------------------
    769788
  • docs/forms.txt

     
    475475   new_data = request.POST.copy()
    476476   new_data.update(request.FILES)
    477477
     478Streaming file uploads.
     479-----------------------
     480
     481File uploads will be read into memory by default. This works fine for
     482small to medium sized uploads (from 1MB to 100MB depending on your
     483setup and usage). If you want to support larger uploads you can enable
     484upload streaming where only a small part of the file will be in memory
     485at any time. To do this you need to specify the ``FILE_UPLOAD_DIR``
     486setting (see the settings_ document for more details).
     487
     488See `request object`_ for more details about ``request.FILES`` objects
     489with streaming file uploads enabled.
     490
    478491Validators
    479492==========
    480493
     
    698711.. _`generic views`: ../generic_views/
    699712.. _`models API`: ../model-api/
    700713.. _settings: ../settings/
     714.. _request object: ../request_response/#httprequest-objects
Back to Top