Code

Ticket #2070: streaming.7106.patch

File streaming.7106.patch, 48.0 KB (added by egon, 6 years ago)

Passes al tests.

  • django/http/multipartparser.py

     
     1""" 
     2MultiPart parsing for file uploads. 
     3If both a progress id is sent (either through ``X-Progress-ID`` 
     4header or ``progress_id`` GET) and ``FILE_UPLOAD_DIR`` is set 
     5in the settings, then the file progress will be tracked using 
     6``request.file_progress``. 
     7 
     8To use this feature, consider creating a middleware with an appropriate 
     9``process_request``:: 
     10 
     11    class FileProgressTrack(object): 
     12        def __get__(self, request, HttpRequest): 
     13            progress_id = request.META['UPLOAD_PROGRESS_ID'] 
     14            status = # get progress from progress_id here 
     15 
     16            return status 
     17 
     18        def __set__(self, request, new_value): 
     19            progress_id = request.META['UPLOAD_PROGRESS_ID'] 
     20 
     21            # set the progress using progress_id here. 
     22 
     23    # example middleware 
     24    class FileProgressExample(object): 
     25        def process_request(self, request): 
     26            request.__class__.file_progress = FileProgressTrack() 
     27 
     28 
     29 
     30""" 
     31 
     32__all__ = ['MultiPartParserError','MultiPartParser'] 
     33 
     34 
     35from django.utils.datastructures import MultiValueDict 
     36from django.http.utils import str_to_unicode 
     37from django.conf import settings 
     38import os 
     39 
     40try: 
     41    from cStringIO import StringIO 
     42except ImportError: 
     43    from StringIO import StringIO 
     44 
     45 
     46class MultiPartParserError(Exception): 
     47    def __init__(self, message): 
     48        self.message = message 
     49    def __str__(self): 
     50        return repr(self.message) 
     51 
     52class MultiPartParser(object): 
     53    """ 
     54    A rfc2388 multipart/form-data parser. 
     55     
     56    parse() reads the input stream in chunk_size chunks and returns a 
     57    tuple of (POST MultiValueDict, FILES MultiValueDict). If 
     58    file_upload_dir is defined files will be streamed to temporary 
     59    files in the specified directory. 
     60 
     61    The FILES dictionary will have 'filename', 'content-type', 
     62    'content' and 'content-length' entries. For streamed files it will 
     63    also have 'tmpfilename' and 'tmpfile'. The 'content' entry will 
     64    only be read from disk when referenced for streamed files. 
     65 
     66    If the X-Progress-ID is sent (in one of many formats), then 
     67    object.file_progress will be given a dictionary of the progress. 
     68    """ 
     69    def __init__(self, headers, input, request, file_upload_dir=None, streaming_min_post_size=None, chunk_size=1024*64): 
     70        try: 
     71            content_length = int(headers['Content-Length']) 
     72        except: 
     73            raise MultiPartParserError('Invalid Content-Length: %s' % headers.get('Content-Length')) 
     74 
     75        content_type = headers.get('Content-Type') 
     76 
     77        if not content_type or not content_type.startswith('multipart/'): 
     78            raise MultiPartParserError('Invalid Content-Type: %s' % content_type) 
     79             
     80        ctype, opts = self.parse_header(content_type) 
     81        boundary = opts.get('boundary') 
     82        from cgi import valid_boundary 
     83        if not boundary or not valid_boundary(boundary): 
     84            raise MultiPartParserError('Invalid boundary in multipart form: %s' % boundary) 
     85 
     86        progress_id = request.META['UPLOAD_PROGRESS_ID'] 
     87 
     88        self._track_progress = file_upload_dir and progress_id # whether or not to track progress 
     89        self._boundary = '--' + boundary 
     90        self._input = input 
     91        self._size = content_length 
     92        self._received = 0 
     93        self._file_upload_dir = file_upload_dir 
     94        self._chunk_size = chunk_size 
     95        self._state = 'PREAMBLE' 
     96        self._partial = '' 
     97        self._post = MultiValueDict() 
     98        self._files = MultiValueDict() 
     99        self._request = request 
     100        self._encoding = request.encoding or settings.DEFAULT_CHARSET 
     101 
     102        if streaming_min_post_size is not None and content_length < streaming_min_post_size: 
     103            self._file_upload_dir = None # disable file streaming for small request 
     104        elif self._track_progress: 
     105            request.file_progress = {'state': 'starting'} 
     106 
     107        try: 
     108            # Use mx fast string search if available. 
     109            from mx.TextTools import FS 
     110            self._fs = FS(self._boundary) 
     111        except ImportError: 
     112            self._fs = None 
     113 
     114    def parse(self): 
     115        try: 
     116            self._parse() 
     117        finally: 
     118            if self._track_progress: 
     119                self._request.file_progress = {'state': 'done'} 
     120        return self._post, self._files 
     121 
     122    def _parse(self): 
     123        size = self._size 
     124 
     125        try: 
     126            while size > 0: 
     127                n = self._read(self._input, min(self._chunk_size, size)) 
     128                if not n: 
     129                    break 
     130                size -= n 
     131        except: 
     132            # consume any remaining data so we dont generate a "Connection Reset" error 
     133            size = self._size - self._received 
     134            while size > 0: 
     135                data = self._input.read(min(self._chunk_size, size)) 
     136                size -= len(data) 
     137            raise 
     138 
     139    def _find_boundary(self, data, start, stop): 
     140        """ 
     141        Find the next boundary and return the end of current part 
     142        and start of next part. 
     143        """ 
     144        if self._fs: 
     145            boundary = self._fs.find(data, start, stop) 
     146        else: 
     147            boundary = data.find(self._boundary, start, stop) 
     148        if boundary >= 0: 
     149            end = boundary 
     150            next = boundary + len(self._boundary) 
     151 
     152            # backup over CRLF 
     153            if end > 0 and data[end-1] == '\n': end -= 1 
     154            if end > 0 and data[end-1] == '\r': end -= 1 
     155            # skip over --CRLF 
     156            if next < stop and data[next] == '-': next += 1 
     157            if next < stop and data[next] == '-': next += 1 
     158            if next < stop and data[next] == '\r': next += 1 
     159            if next < stop and data[next] == '\n': next += 1 
     160 
     161            return True, end, next 
     162        else: 
     163            return False, stop, stop 
     164 
     165    class TemporaryFile(object): 
     166        "A temporary file that tries to delete itself when garbage collected." 
     167        def __init__(self, dir): 
     168            import tempfile 
     169            (fd, name) = tempfile.mkstemp(suffix='.upload', dir=dir) 
     170            self.file = os.fdopen(fd, 'w+b') 
     171            self.name = name 
     172 
     173        def __getattr__(self, name): 
     174            a = getattr(self.__dict__['file'], name) 
     175            if type(a) != type(0): 
     176                setattr(self, name, a) 
     177            return a 
     178 
     179        def __del__(self): 
     180            try: 
     181                os.unlink(self.name) 
     182            except OSError: 
     183                pass 
     184 
     185    class LazyContent(dict): 
     186        """ 
     187        A lazy FILES dictionary entry that reads the contents from 
     188        tmpfile only when referenced. 
     189        """ 
     190        def __init__(self, data): 
     191            dict.__init__(self, data) 
     192 
     193        def __getitem__(self, key): 
     194            if key == 'content' and not self.has_key(key): 
     195                self['tmpfile'].seek(0) 
     196                self['content'] = self['tmpfile'].read() 
     197            return dict.__getitem__(self, key) 
     198 
     199    def _read(self, input, size): 
     200        data = input.read(size) 
     201 
     202        if not data: 
     203            return 0 
     204 
     205        read_size = len(data) 
     206        self._received += read_size 
     207 
     208        if self._partial: 
     209            data = self._partial + data 
     210 
     211        start = 0 
     212        stop = len(data) 
     213 
     214        while start < stop: 
     215            boundary, end, next = self._find_boundary(data, start, stop) 
     216 
     217            if not boundary and read_size: 
     218                # make sure we dont treat a partial boundary (and its separators) as data 
     219                stop -= len(self._boundary) + 16 
     220                end = next = stop 
     221                if end <= start: 
     222                    break # need more data 
     223 
     224            if self._state == 'PREAMBLE': 
     225                # Preamble, just ignore it 
     226                self._state = 'HEADER' 
     227 
     228            elif self._state == 'HEADER': 
     229                # Beginning of header, look for end of header and parse it if found. 
     230 
     231                header_end = data.find('\r\n\r\n', start, stop) 
     232                if header_end == -1: 
     233                    break # need more data 
     234 
     235                header = data[start:header_end] 
     236 
     237                self._fieldname = None 
     238                self._filename = None 
     239                self._content_type = None 
     240 
     241                for line in header.split('\r\n'): 
     242                    ctype, opts = self.parse_header(line) 
     243                    if ctype == 'content-disposition: form-data': 
     244                        self._fieldname = opts.get('name') 
     245                        self._filename = opts.get('filename') 
     246                    elif ctype.startswith('content-type: '): 
     247                        self._content_type = ctype[14:] 
     248 
     249                if self._filename is not None: 
     250                    # cleanup filename from IE full paths: 
     251                    self._filename = self._filename[self._filename.rfind("\\")+1:].strip() 
     252 
     253                    if self._filename: # ignore files without filenames 
     254                        if self._file_upload_dir: 
     255                            try: 
     256                                self._file = self.TemporaryFile(dir=self._file_upload_dir) 
     257                            except (OSError, IOError), e: 
     258                                raise MultiPartParserError("Failed to create temporary file. Error was %s" % e) 
     259                        else: 
     260                            self._file = StringIO() 
     261                    else: 
     262                        self._file = None 
     263                    self._filesize = 0 
     264                    self._state = 'FILE' 
     265                else: 
     266                    self._field = StringIO() 
     267                    self._state = 'FIELD' 
     268                next = header_end + 4 
     269 
     270            elif self._state == 'FIELD': 
     271                # In a field, collect data until a boundary is found. 
     272 
     273                self._field.write(data[start:end]) 
     274                if boundary: 
     275                    if self._fieldname: 
     276                        self._post.appendlist(self._fieldname, str_to_unicode(self._field.getvalue(), self._encoding)) 
     277                    self._field.close() 
     278                    self._state = 'HEADER' 
     279 
     280            elif self._state == 'FILE': 
     281                # In a file, collect data until a boundary is found. 
     282 
     283                if self._file: 
     284                    try: 
     285                        self._file.write(data[start:end]) 
     286                    except IOError, e: 
     287                        raise MultiPartParserError("Failed to write to temporary file.") 
     288                    self._filesize += end-start 
     289 
     290                    if self._track_progress: 
     291                        self._request.file_progress = {'received': self._received, 
     292                                                       'size':     self._size, 
     293                                                       'state':    'uploading'} 
     294 
     295                if boundary: 
     296                    if self._file: 
     297                        if self._file_upload_dir: 
     298                            self._file.seek(0) 
     299                            file = self.LazyContent({ 
     300                                'filename': str_to_unicode(self._filename, self._encoding), 
     301                                'content-type':  self._content_type, 
     302                                # 'content': is read on demand 
     303                                'content-length': self._filesize, 
     304                                'tmpfilename': self._file.name, 
     305                                'tmpfile': self._file 
     306                            }) 
     307                        else: 
     308                            file = { 
     309                                'filename': str_to_unicode(self._filename, self._encoding), 
     310                                'content-type':  self._content_type, 
     311                                'content': self._file.getvalue(), 
     312                                'content-length': self._filesize 
     313                            } 
     314                            self._file.close() 
     315 
     316                        self._files.appendlist(self._fieldname, file) 
     317 
     318                    self._state = 'HEADER' 
     319 
     320            start = next 
     321 
     322        self._partial = data[start:] 
     323 
     324        return read_size 
     325 
     326    def parse_header(self, line): 
     327        from cgi import parse_header 
     328        return parse_header(line) 
  • django/http/__init__.py

     
    11import os 
     2import re 
    23from Cookie import SimpleCookie 
    34from pprint import pformat 
    45from urllib import urlencode 
    56from urlparse import urljoin 
     7from django.http.utils import str_to_unicode 
     8from django.http.multipartparser import MultiPartParser, MultiPartParserError 
    69from django.utils.datastructures import MultiValueDict, FileDict 
    710from django.utils.encoding import smart_str, iri_to_uri, force_unicode 
    811from utils import * 
    912 
     13upload_id_re = re.compile(r'^[a-fA-F0-9]{32}$') # file progress id Regular expression 
     14 
    1015RESERVED_CHARS="!*'();:@&=+$,/?%#[]" 
    1116 
    1217try: 
     
    7984 
    8085    def is_secure(self): 
    8186        return os.environ.get("HTTPS") == "on" 
    82  
     87         
    8388    def _set_encoding(self, val): 
    8489        """ 
    8590        Sets the encoding used for GET/POST accesses. If the GET or POST 
     
    97102 
    98103    encoding = property(_get_encoding, _set_encoding) 
    99104 
    100 def parse_file_upload(header_dict, post_data): 
    101     "Returns a tuple of (POST QueryDict, FILES MultiValueDict)" 
    102     import email, email.Message 
    103     from cgi import parse_header 
    104     raw_message = '\r\n'.join(['%s:%s' % pair for pair in header_dict.items()]) 
    105     raw_message += '\r\n\r\n' + post_data 
    106     msg = email.message_from_string(raw_message) 
    107     POST = QueryDict('', mutable=True) 
    108     FILES = MultiValueDict() 
    109     for submessage in msg.get_payload(): 
    110         if submessage and isinstance(submessage, email.Message.Message): 
    111             name_dict = parse_header(submessage['Content-Disposition'])[1] 
    112             # name_dict is something like {'name': 'file', 'filename': 'test.txt'} for file uploads 
    113             # or {'name': 'blah'} for POST fields 
    114             # We assume all uploaded files have a 'filename' set. 
    115             if 'filename' in name_dict: 
    116                 assert type([]) != type(submessage.get_payload()), "Nested MIME messages are not supported" 
    117                 if not name_dict['filename'].strip(): 
    118                     continue 
    119                 # IE submits the full path, so trim everything but the basename. 
    120                 # (We can't use os.path.basename because that uses the server's 
    121                 # directory separator, which may not be the same as the 
    122                 # client's one.) 
    123                 filename = name_dict['filename'][name_dict['filename'].rfind("\\")+1:] 
    124                 FILES.appendlist(name_dict['name'], FileDict({ 
    125                     'filename': filename, 
    126                     'content-type': 'Content-Type' in submessage and submessage['Content-Type'] or None, 
    127                     'content': submessage.get_payload(), 
    128                 })) 
    129             else: 
    130                 POST.appendlist(name_dict['name'], submessage.get_payload()) 
    131     return POST, FILES 
     105    def _get_file_progress(self): 
     106        return {} 
     107     
     108    def _set_file_progress(self,value): 
     109        pass 
    132110 
     111    def _del_file_progress(self): 
     112        pass 
     113 
     114    file_progress = property(_get_file_progress, 
     115                             _set_file_progress, 
     116                             _del_file_progress) 
     117 
     118    def _get_file_progress_from_args(self, headers, get, querystring): 
     119        """ 
     120        This parses the request for a file progress_id value. 
     121        Note that there are two distinct ways of getting the progress 
     122        ID -- header and GET. One is used primarily to attach via JavaScript 
     123        to the end of an HTML form action while the other is used for AJAX 
     124        communication. 
     125 
     126        All progress IDs must be valid 32-digit hexadecimal numbers. 
     127        """ 
     128        if 'X-Upload-ID' in headers: 
     129            progress_id = headers['X-Upload-ID'] 
     130        elif 'progress_id' in get: 
     131            progress_id = get['progress_id'] 
     132        else: 
     133            return None 
     134 
     135        if not upload_id_re.match(progress_id): 
     136            return None 
     137 
     138        return progress_id 
     139 
     140def parse_file_upload(headers, input, request): 
     141    from django.conf import settings 
     142 
     143    # Only stream files to disk if FILE_STREAMING_DIR is set 
     144    file_upload_dir = settings.FILE_UPLOAD_DIR 
     145    streaming_min_post_size = settings.STREAMING_MIN_POST_SIZE 
     146 
     147    try: 
     148        parser = MultiPartParser(headers, input, request, file_upload_dir, streaming_min_post_size) 
     149        return parser.parse() 
     150    except MultiPartParserError, e: 
     151        return MultiValueDict({ '_file_upload_error': [e.message] }), {} 
     152 
     153 
    133154class QueryDict(MultiValueDict): 
    134155    """ 
    135156    A specialized MultiValueDict that takes a query string when initialized. 
     
    413434# A backwards compatible alias for HttpRequest.get_host. 
    414435def get_host(request): 
    415436    return request.get_host() 
    416  
    417 # It's neither necessary nor appropriate to use 
    418 # django.utils.encoding.smart_unicode for parsing URLs and form inputs. Thus, 
    419 # this slightly more restricted function. 
    420 def str_to_unicode(s, encoding): 
    421     """ 
    422     Convert basestring objects to unicode, using the given encoding. Illegaly 
    423     encoded input characters are replaced with Unicode "unknown" codepoint 
    424     (\ufffd). 
    425  
    426     Returns any non-basestring objects without change. 
    427     """ 
    428     if isinstance(s, str): 
    429         return unicode(s, encoding, 'replace') 
    430     else: 
    431         return s 
    432  
  • django/http/utils.py

     
     1# It's neither necessary nor appropriate to use 
     2# django.utils.encoding.smart_unicode for parsing URLs and form inputs. Thus, 
     3# this slightly more restricted function. 
     4def str_to_unicode(s, encoding): 
     5    """ 
     6    Convert basestring objects to unicode, using the given encoding. Illegaly 
     7    encoded input characters are replaced with Unicode "unknown" codepoint 
     8    (\ufffd). 
     9 
     10    Returns any non-basestring objects without change. 
     11    """ 
     12    if isinstance(s, str): 
     13        return unicode(s, encoding, 'replace') 
     14    else: 
     15        return s 
     16 
    117""" 
    218Functions that modify an HTTP request or response in some way. 
    319""" 
  • django/conf/global_settings.py

     
    257257DEFAULT_TABLESPACE = '' 
    258258DEFAULT_INDEX_TABLESPACE = '' 
    259259 
     260# The directory to place streamed file uploads. The web server needs write 
     261# permissions on this directory. 
     262# If this is None, streaming uploads are disabled. 
     263FILE_UPLOAD_DIR = None 
     264 
     265# The minimum size of a POST before file uploads are streamed to disk. 
     266# Any less than this number, and the file is uploaded to memory. 
     267# Size is in bytes. 
     268STREAMING_MIN_POST_SIZE = 512 * (2**10) 
     269 
    260270############## 
    261271# MIDDLEWARE # 
    262272############## 
  • django/db/models/base.py

     
    1212from django.dispatch import dispatcher 
    1313from django.utils.datastructures import SortedDict 
    1414from django.utils.functional import curry 
     15from django.utils.file import file_move_safe 
    1516from django.utils.encoding import smart_str, force_unicode, smart_unicode 
    1617from django.conf import settings 
    1718from itertools import izip 
     
    379380    def _get_FIELD_size(self, field): 
    380381        return os.path.getsize(self._get_FIELD_filename(field)) 
    381382 
    382     def _save_FIELD_file(self, field, filename, raw_contents, save=True): 
     383    def _save_FIELD_file(self, field, filename, raw_field, save=True): 
    383384        directory = field.get_directory_name() 
    384385        try: # Create the date-based directory if it doesn't exist. 
    385386            os.makedirs(os.path.join(settings.MEDIA_ROOT, directory)) 
    386387        except OSError: # Directory probably already exists. 
    387388            pass 
     389 
     390        if filename is None: 
     391            filename = raw_field['filename'] 
     392 
    388393        filename = field.get_filename(filename) 
    389394 
    390395        # If the filename already exists, keep adding an underscore to the name of 
     
    401406        setattr(self, field.attname, filename) 
    402407 
    403408        full_filename = self._get_FIELD_filename(field) 
    404         fp = open(full_filename, 'wb') 
    405         fp.write(raw_contents) 
    406         fp.close() 
     409        if raw_field.has_key('tmpfilename'): 
     410            raw_field['tmpfile'].close() 
     411            file_move_safe(raw_field['tmpfilename'], full_filename) 
     412        else: 
     413            from django.utils import file_locks 
     414            fp = open(full_filename, 'wb') 
     415            # exclusive lock 
     416            file_locks.lock(fp, file_locks.LOCK_EX) 
     417            fp.write(raw_field['content']) 
     418            fp.close() 
    407419 
    408420        # Save the width and/or height, if applicable. 
    409421        if isinstance(field, ImageField) and (field.width_field or field.height_field): 
  • django/db/models/fields/__init__.py

     
    761761        setattr(cls, 'get_%s_filename' % self.name, curry(cls._get_FIELD_filename, field=self)) 
    762762        setattr(cls, 'get_%s_url' % self.name, curry(cls._get_FIELD_url, field=self)) 
    763763        setattr(cls, 'get_%s_size' % self.name, curry(cls._get_FIELD_size, field=self)) 
    764         setattr(cls, 'save_%s_file' % self.name, lambda instance, filename, raw_contents, save=True: instance._save_FIELD_file(self, filename, raw_contents, save)) 
     764        setattr(cls, 'save_%s_file' % self.name, lambda instance, filename, raw_field, save=True: instance._save_FIELD_file(self, filename, raw_field, save)) 
     765        setattr(cls, 'move_%s_file' % self.name, lambda instance, raw_field, save=True: instance._save_FIELD_file(self, None, raw_field, save))         
    765766        dispatcher.connect(self.delete_file, signal=signals.post_delete, sender=cls) 
    766767 
    767768    def delete_file(self, instance): 
     
    784785        if new_data.get(upload_field_name, False): 
    785786            func = getattr(new_object, 'save_%s_file' % self.name) 
    786787            if rel: 
    787                 func(new_data[upload_field_name][0]["filename"], new_data[upload_field_name][0]["content"], save) 
     788                func(new_data[upload_field_name][0]["filename"], new_data[upload_field_name][0], save) 
    788789            else: 
    789                 func(new_data[upload_field_name]["filename"], new_data[upload_field_name]["content"], save) 
     790                func(new_data[upload_field_name]["filename"], new_data[upload_field_name], save) 
    790791 
    791792    def get_directory_name(self): 
    792793        return os.path.normpath(force_unicode(datetime.datetime.now().strftime(smart_str(self.upload_to)))) 
     
    799800    def save_form_data(self, instance, data): 
    800801        from django.newforms.fields import UploadedFile 
    801802        if data and isinstance(data, UploadedFile): 
    802             getattr(instance, "save_%s_file" % self.name)(data.filename, data.content, save=False) 
     803            getattr(instance, "save_%s_file" % self.name)(data.filename, data.data, save=False) 
    803804 
    804805    def formfield(self, **kwargs): 
    805806        defaults = {'form_class': forms.FileField} 
  • django/oldforms/__init__.py

     
    681681        self.validator_list = [self.isNonEmptyFile] + validator_list 
    682682 
    683683    def isNonEmptyFile(self, field_data, all_data): 
    684         try: 
    685             content = field_data['content'] 
    686         except TypeError: 
     684        if field_data.has_key('_file_upload_error'): 
     685            raise validators.CriticalValidationError, field_data['_file_upload_error'] 
     686        if not field_data.has_key('filename'): 
    687687            raise validators.CriticalValidationError, ugettext("No file was submitted. Check the encoding type on the form.") 
    688         if not content: 
     688        if not field_data['content-length']: 
    689689            raise validators.CriticalValidationError, ugettext("The submitted file is empty.") 
    690690 
    691691    def render(self, data): 
    692692        return mark_safe(u'<input type="file" id="%s" class="v%s" name="%s" />' % \ 
    693693            (self.get_id(), self.__class__.__name__, self.field_name)) 
    694694 
     695    def prepare(self, new_data): 
     696        if new_data.has_key('_file_upload_error'): 
     697            # pretend we got something in the field to raise a validation error later 
     698            new_data[self.field_name] = { '_file_upload_error': new_data['_file_upload_error'] } 
     699 
    695700    def html2python(data): 
    696701        if data is None: 
    697702            raise EmptyValue 
  • django/core/validators.py

     
    177177    from PIL import Image 
    178178    from cStringIO import StringIO 
    179179    try: 
    180         content = field_data['content'] 
     180        filename = field_data['filename'] 
    181181    except TypeError: 
    182182        raise ValidationError, _("No file was submitted. Check the encoding type on the form.") 
    183183    try: 
    184184        # load() is the only method that can spot a truncated JPEG, 
    185185        #  but it cannot be called sanely after verify() 
    186         trial_image = Image.open(StringIO(content)) 
     186        trial_image = Image.open(field_data.get('tmpfilename') or StringIO(field_data.get('content',''))) 
    187187        trial_image.load() 
    188188        # verify() is the only method that can spot a corrupt PNG, 
    189189        #  but it must be called immediately after the constructor 
    190         trial_image = Image.open(StringIO(content)) 
     190        trial_image = Image.open(field_data.get('tmpfilename') or StringIO(field_data.get('content',''))) 
    191191        trial_image.verify() 
    192192    except Exception: # Python Imaging Library doesn't recognize it as an image 
    193193        raise ValidationError, _("Upload a valid image. The file you uploaded was either not an image or a corrupted image.") 
  • django/core/handlers/wsgi.py

     
    7777        self.environ = environ 
    7878        self.path = force_unicode(environ['PATH_INFO']) 
    7979        self.META = environ 
     80        self.META['UPLOAD_PROGRESS_ID'] = self._get_file_progress_id() 
    8081        self.method = environ['REQUEST_METHOD'].upper() 
    8182 
    8283    def __repr__(self): 
     
    114115            if self.environ.get('CONTENT_TYPE', '').startswith('multipart'): 
    115116                header_dict = dict([(k, v) for k, v in self.environ.items() if k.startswith('HTTP_')]) 
    116117                header_dict['Content-Type'] = self.environ.get('CONTENT_TYPE', '') 
    117                 self._post, self._files = http.parse_file_upload(header_dict, self.raw_post_data) 
     118                header_dict['Content-Length'] = self.environ.get('CONTENT_LENGTH', '') 
     119                header_dict['X-Progress-ID'] = self.environ.get('HTTP_X_PROGRESS_ID', '') 
     120                try: 
     121                    self._post, self._files = http.parse_file_upload(header_dict, self.environ['wsgi.input'], self) 
     122                except: 
     123                    self._post, self._files = {}, {} # make sure we dont read the input stream again 
     124                    raise 
     125                self._raw_post_data = None # raw data is not available for streamed multipart messages 
    118126            else: 
    119127                self._post, self._files = http.QueryDict(self.raw_post_data, encoding=self._encoding), datastructures.MultiValueDict() 
    120128        else: 
     
    172180            buf.close() 
    173181            return self._raw_post_data 
    174182 
     183    def _get_file_progress_id(self): 
     184        """ 
     185        Returns the Progress ID of the request, 
     186        usually provided if there is a file upload 
     187        going on. 
     188        Returns ``None`` if no progress ID is specified. 
     189        """ 
     190        return self._get_file_progress_from_args(self.environ, 
     191                                                 self.GET, 
     192                                                 self.environ.get('QUERY_STRING', '')) 
     193 
    175194    GET = property(_get_get, _set_get) 
    176195    POST = property(_get_post, _set_post) 
    177196    COOKIES = property(_get_cookies, _set_cookies) 
  • django/core/handlers/modpython.py

     
    5252    def _load_post_and_files(self): 
    5353        "Populates self._post and self._files" 
    5454        if 'content-type' in self._req.headers_in and self._req.headers_in['content-type'].startswith('multipart'): 
    55             self._post, self._files = http.parse_file_upload(self._req.headers_in, self.raw_post_data) 
     55            self._raw_post_data = None # raw data is not available for streamed multipart messages 
     56            try: 
     57                self._post, self._files = http.parse_file_upload(self._req.headers_in, self._req, self) 
     58            except: 
     59                self._post, self._files = {}, {} # make sure we dont read the input stream again 
     60                raise 
    5661        else: 
    5762            self._post, self._files = http.QueryDict(self.raw_post_data, encoding=self._encoding), datastructures.MultiValueDict() 
    5863 
     
    97102                'AUTH_TYPE':         self._req.ap_auth_type, 
    98103                'CONTENT_LENGTH':    self._req.clength, # This may be wrong 
    99104                'CONTENT_TYPE':      self._req.content_type, # This may be wrong 
    100                 'GATEWAY_INTERFACE': 'CGI/1.1', 
    101                 'PATH_INFO':         self._req.path_info, 
    102                 'PATH_TRANSLATED':   None, # Not supported 
    103                 'QUERY_STRING':      self._req.args, 
    104                 'REMOTE_ADDR':       self._req.connection.remote_ip, 
    105                 'REMOTE_HOST':       None, # DNS lookups not supported 
    106                 'REMOTE_IDENT':      self._req.connection.remote_logname, 
    107                 'REMOTE_USER':       self._req.user, 
    108                 'REQUEST_METHOD':    self._req.method, 
    109                 'SCRIPT_NAME':       None, # Not supported 
    110                 'SERVER_NAME':       self._req.server.server_hostname, 
    111                 'SERVER_PORT':       self._req.server.port, 
    112                 'SERVER_PROTOCOL':   self._req.protocol, 
    113                 'SERVER_SOFTWARE':   'mod_python' 
     105                'GATEWAY_INTERFACE':  'CGI/1.1', 
     106                'PATH_INFO':          self._req.path_info, 
     107                'PATH_TRANSLATED':    None, # Not supported 
     108                'QUERY_STRING':       self._req.args, 
     109                'REMOTE_ADDR':        self._req.connection.remote_ip, 
     110                'REMOTE_HOST':        None, # DNS lookups not supported 
     111                'REMOTE_IDENT':       self._req.connection.remote_logname, 
     112                'REMOTE_USER':        self._req.user, 
     113                'REQUEST_METHOD':     self._req.method, 
     114                'SCRIPT_NAME':        None, # Not supported 
     115                'SERVER_NAME':        self._req.server.server_hostname, 
     116                'SERVER_PORT':        self._req.server.port, 
     117                'SERVER_PROTOCOL':    self._req.protocol, 
     118                'UPLOAD_PROGRESS_ID': self._get_file_progress_id(), 
     119                'SERVER_SOFTWARE':    'mod_python' 
    114120            } 
    115121            for key, value in self._req.headers_in.items(): 
    116122                key = 'HTTP_' + key.upper().replace('-', '_') 
     
    127133    def _get_method(self): 
    128134        return self.META['REQUEST_METHOD'].upper() 
    129135 
     136    def _get_file_progress_id(self): 
     137        """ 
     138        Returns the Progress ID of the request, 
     139        usually provided if there is a file upload 
     140        going on. 
     141        Returns ``None`` if no progress ID is specified. 
     142        """ 
     143        return self._get_file_progress_from_args(self._req.headers_in, 
     144                                                 self.GET, 
     145                                                 self._req.args) 
     146 
    130147    GET = property(_get_get, _set_get) 
    131148    POST = property(_get_post, _set_post) 
    132149    COOKIES = property(_get_cookies, _set_cookies) 
  • django/utils/file_locks.py

     
     1""" 
     2Locking portability by Jonathan Feignberg <jdf@pobox.com> in python cookbook 
     3 
     4Example Usage:: 
     5 
     6    from django.utils import file_locks 
     7 
     8    f = open('./file', 'wb') 
     9 
     10    file_locks.lock(f, file_locks.LOCK_EX) 
     11    f.write('Django') 
     12    f.close() 
     13""" 
     14 
     15 
     16import os 
     17 
     18__all__ = ['LOCK_EX','LOCK_SH','LOCK_NB','lock','unlock'] 
     19 
     20if os.name == 'nt': 
     21        import win32con 
     22        import win32file 
     23        import pywintypes 
     24        LOCK_EX = win32con.LOCKFILE_EXCLUSIVE_LOCK 
     25        LOCK_SH = 0 
     26        LOCK_NB = win32con.LOCKFILE_FAIL_IMMEDIATELY 
     27        __overlapped = pywintypes.OVERLAPPED() 
     28elif os.name == 'posix': 
     29        import fcntl 
     30        LOCK_EX = fcntl.LOCK_EX 
     31        LOCK_SH = fcntl.LOCK_SH 
     32        LOCK_NB = fcntl.LOCK_NB 
     33else: 
     34        raise RuntimeError("Locking only defined for nt and posix platforms") 
     35 
     36if os.name == 'nt': 
     37        def lock(file, flags): 
     38                hfile = win32file._get_osfhandle(file.fileno()) 
     39                win32file.LockFileEx(hfile, flags, 0, -0x10000, __overlapped) 
     40 
     41        def unlock(file): 
     42                hfile = win32file._get_osfhandle(file.fileno()) 
     43                win32file.UnlockFileEx(hfile, 0, -0x10000, __overlapped) 
     44 
     45elif os.name =='posix': 
     46        def lock(file, flags): 
     47                fcntl.flock(file.fileno(), flags) 
     48 
     49        def unlock(file): 
     50                fcntl.flock(file.fileno(), fcntl.LOCK_UN) 
  • django/utils/file.py

     
     1import os 
     2 
     3__all__ = ['file_move_safe'] 
     4 
     5try: 
     6    import shutil 
     7    file_move = shutil.move 
     8except ImportError: 
     9    file_move = os.rename 
     10 
     11def file_move_safe(old_file_name, new_file_name, chunk_size = 1024*64, allow_overwrite=False): 
     12    """ 
     13    Moves a file from one location to another in the safest way possible. 
     14     
     15    First, it tries using shutils.move, which is OS-dependent but doesn't 
     16    break with change of filesystems. Then it tries os.rename, which will 
     17    break if it encounters a change in filesystems. Lastly, it streams 
     18    it manually from one file to another in python. 
     19 
     20    Without ``allow_overwrite``, if the destination file exists, the 
     21    file will raise an IOError. 
     22    """ 
     23 
     24    from django.utils import file_locks 
     25 
     26    if old_file_name == new_file_name: 
     27        # No file moving takes place. 
     28        return 
     29 
     30    if not allow_overwrite and os.path.exists(new_file_name): 
     31        raise IOError, "Django does not allow overwriting files." 
     32 
     33    try: 
     34        file_move(old_file_name, new_file_name) 
     35        return 
     36    except OSError: # moving to another filesystem 
     37        pass 
     38 
     39    new_file = open(new_file_name, 'wb') 
     40    # exclusive lock 
     41    file_locks.lock(new_file, file_locks.LOCK_EX) 
     42    old_file = open(old_file_name, 'rb') 
     43    current_chunk = None 
     44 
     45    while current_chunk != '': 
     46        current_chunk = old_file.read(chunk_size) 
     47        new_file.write(current_chunk) 
     48 
     49    new_file.close() 
     50    old_file.close() 
     51 
     52    os.remove(old_file_name) 
     53 
  • django/newforms/fields.py

     
    415415 
    416416class UploadedFile(StrAndUnicode): 
    417417    "A wrapper for files uploaded in a FileField" 
    418     def __init__(self, filename, content): 
     418    def __init__(self, filename, data): 
    419419        self.filename = filename 
    420         self.content = content 
     420        self.data = data  
    421421 
    422422    def __unicode__(self): 
    423423        """ 
     
    444444        elif not data and initial: 
    445445            return initial 
    446446        try: 
    447             f = UploadedFile(data['filename'], data['content']) 
     447            f = UploadedFile(data['filename'], data) 
    448448        except TypeError: 
    449449            raise ValidationError(self.error_messages['invalid']) 
    450450        except KeyError: 
    451451            raise ValidationError(self.error_messages['missing']) 
    452         if not f.content: 
     452        if not f.data.get('content-length'): 
    453453            raise ValidationError(self.error_messages['empty']) 
    454454        return f 
    455455 
     
    473473        try: 
    474474            # load() is the only method that can spot a truncated JPEG, 
    475475            #  but it cannot be called sanely after verify() 
    476             trial_image = Image.open(StringIO(f.content)) 
     476            trial_image = Image.open(f.data.get('tmpfilename') or StringIO(f.data['content'])) 
    477477            trial_image.load() 
    478478            # verify() is the only method that can spot a corrupt PNG, 
    479479            #  but it must be called immediately after the constructor 
    480             trial_image = Image.open(StringIO(f.content)) 
     480            trial_image = Image.open(f.data.get('tmpfilename') or StringIO(f.data['content'])) 
    481481            trial_image.verify() 
    482482        except Exception: # Python Imaging Library doesn't recognize it as an image 
    483483            raise ValidationError(self.error_messages['invalid_image']) 
  • tests/modeltests/model_forms/models.py

     
    736736 
    737737# Upload a file and ensure it all works as expected. 
    738738 
    739 >>> f = TextFileForm(data={'description': u'Assistance'}, files={'file': {'filename': 'test1.txt', 'content': 'hello world'}}) 
     739>>> f = TextFileForm(data={'description': u'Assistance'}, files={'file': {'filename': 'test1.txt', 'content': 'hello world', 'content-length': len('hello world')}}) 
    740740>>> f.is_valid() 
    741741True 
    742742>>> type(f.cleaned_data['file']) 
     
    763763 
    764764# Override the file by uploading a new one. 
    765765 
    766 >>> f = TextFileForm(data={'description': u'Assistance'}, files={'file': {'filename': 'test2.txt', 'content': 'hello world'}}, instance=instance) 
     766>>> f = TextFileForm(data={'description': u'Assistance'}, files={'file': {'filename': 'test2.txt', 'content': 'hello world', 'content-length': len('hello world')}}, instance=instance) 
    767767>>> f.is_valid() 
    768768True 
    769769>>> instance = f.save() 
     
    782782>>> instance.file 
    783783'' 
    784784 
    785 >>> f = TextFileForm(data={'description': u'Assistance'}, files={'file': {'filename': 'test3.txt', 'content': 'hello world'}}, instance=instance) 
     785>>> f = TextFileForm(data={'description': u'Assistance'}, files={'file': {'filename': 'test3.txt', 'content': 'hello world', 'content-length': len('hello world')}}, instance=instance) 
    786786>>> f.is_valid() 
    787787True 
    788788>>> instance = f.save() 
     
    802802 
    803803>>> image_data = open(os.path.join(os.path.dirname(__file__), "test.png")).read() 
    804804 
    805 >>> f = ImageFileForm(data={'description': u'An image'}, files={'image': {'filename': 'test.png', 'content': image_data}}) 
     805>>> f = ImageFileForm(data={'description': u'An image'}, files={'image': {'filename': 'test.png', 'content': image_data, 'content-length': len(image_data)}}) 
    806806>>> f.is_valid() 
    807807True 
    808808>>> type(f.cleaned_data['image']) 
     
    829829 
    830830# Override the file by uploading a new one. 
    831831 
    832 >>> f = ImageFileForm(data={'description': u'Changed it'}, files={'image': {'filename': 'test2.png', 'content': image_data}}, instance=instance) 
     832>>> f = ImageFileForm(data={'description': u'Changed it'}, files={'image': {'filename': 'test2.png', 'content': image_data, 'content-length': len(image_data)}}, instance=instance) 
    833833>>> f.is_valid() 
    834834True 
    835835>>> instance = f.save() 
     
    848848>>> instance.image 
    849849'' 
    850850 
    851 >>> f = ImageFileForm(data={'description': u'And a final one'}, files={'image': {'filename': 'test3.png', 'content': image_data}}, instance=instance) 
     851>>> f = ImageFileForm(data={'description': u'And a final one'}, files={'image': {'filename': 'test3.png', 'content': image_data, 'content-length': len(image_data)}}, instance=instance) 
    852852>>> f.is_valid() 
    853853True 
    854854>>> instance = f.save() 
  • tests/modeltests/test_client/views.py

     
    4747 
    4848    return HttpResponse(t.render(c)) 
    4949 
     50def post_file_view(request): 
     51    "A view that expects a multipart post and returns a file in the context" 
     52    t = Template('File {{ file.filename }} received', name='POST Template') 
     53    c = Context({'file': request.FILES['file_file']}) 
     54    return HttpResponse(t.render(c)) 
     55 
    5056def redirect_view(request): 
    5157    "A view that redirects all requests to the GET view" 
    5258    if request.GET: 
  • tests/modeltests/test_client/models.py

     
    8080        self.assertEqual(response.template.name, "Book template") 
    8181        self.assertEqual(response.content, "Blink - Malcolm Gladwell") 
    8282 
     83    def test_post_file_view(self): 
     84        "POST this python file to a view" 
     85        import os, tempfile 
     86        from django.conf import settings 
     87        file = __file__.replace('.pyc', '.py') 
     88        for upload_dir, streaming_size in [(None,512*1000), (tempfile.gettempdir(), 1)]: 
     89            settings.FILE_UPLOAD_DIR = upload_dir 
     90            settings.STREAMING_MIN_POST_SIZE = streaming_size 
     91            post_data = { 'name': file, 'file_file': open(file) } 
     92            response = self.client.post('/test_client/post_file_view/', post_data) 
     93            self.failUnless('models.py' in response.context['file']['filename']) 
     94            self.failUnless(len(response.context['file']['content']) == os.path.getsize(file)) 
     95            if upload_dir: 
     96                self.failUnless(response.context['file']['tmpfilename']) 
     97 
    8398    def test_redirect(self): 
    8499        "GET a URL that redirects elsewhere" 
    85100        response = self.client.get('/test_client/redirect_view/') 
  • tests/modeltests/test_client/urls.py

     
    55urlpatterns = patterns('', 
    66    (r'^get_view/$', views.get_view), 
    77    (r'^post_view/$', views.post_view), 
     8    (r'^post_file_view/$', views.post_file_view), 
    89    (r'^raw_post_view/$', views.raw_post_view), 
    910    (r'^redirect_view/$', views.redirect_view), 
    1011    (r'^permanent_redirect_view/$', redirect_to, { 'url': '/test_client/get_view/' }), 
  • tests/regressiontests/forms/fields.py

     
    788788... 
    789789ValidationError: [u'No file was submitted. Check the encoding type on the form.'] 
    790790 
    791 >>> f.clean({'filename': 'name', 'content': None}) 
     791>>> f.clean({'filename': 'name', 'content': None, 'content-length': 0}) 
    792792Traceback (most recent call last): 
    793793... 
    794794ValidationError: [u'The submitted file is empty.'] 
    795795 
    796 >>> f.clean({'filename': 'name', 'content': ''}) 
     796>>> f.clean({'filename': 'name', 'content': '', 'content-length': 0}) 
    797797Traceback (most recent call last): 
    798798... 
    799799ValidationError: [u'The submitted file is empty.'] 
    800800 
    801 >>> type(f.clean({'filename': 'name', 'content': 'Some File Content'})) 
     801>>> type(f.clean({'filename': 'name', 'content': 'Some File Content', 'content-length': len('Some File Content')})) 
    802802<class 'django.newforms.fields.UploadedFile'> 
    803803 
    804 >>> type(f.clean({'filename': 'name', 'content': 'Some File Content'}, 'files/test4.pdf')) 
     804>>> type(f.clean({'filename': 'name', 'content': 'Some File Content', 'content-length': len('Some File Content')}, 'files/test4.pdf')) 
    805805<class 'django.newforms.fields.UploadedFile'> 
    806806 
    807807# URLField ################################################################## 
  • tests/regressiontests/forms/forms.py

     
    14101410>>> print f 
    14111411<tr><th>File1:</th><td><ul class="errorlist"><li>No file was submitted. Check the encoding type on the form.</li></ul><input type="file" name="file1" /></td></tr> 
    14121412 
    1413 >>> f = FileForm(data={}, files={'file1': {'filename': 'name', 'content':'some content'}}, auto_id=False) 
     1413>>> f = FileForm(data={}, files={'file1': {'filename': 'name', 'content':'some content', 'content-length': len('some content')}}, auto_id=False) 
    14141414>>> print f 
    14151415<tr><th>File1:</th><td><input type="file" name="file1" /></td></tr> 
    14161416>>> f.is_valid() 
  • docs/request_response.txt

     
    8282``FILES`` 
    8383    A dictionary-like object containing all uploaded files. Each key in 
    8484    ``FILES`` is the ``name`` from the ``<input type="file" name="" />``. Each 
    85     value in ``FILES`` is a standard Python dictionary with the following three 
     85    value in ``FILES`` is a standard Python dictionary with the following four 
    8686    keys: 
    8787 
    8888        * ``filename`` -- The name of the uploaded file, as a Python string. 
    8989        * ``content-type`` -- The content type of the uploaded file. 
    9090        * ``content`` -- The raw content of the uploaded file. 
     91        * ``content-length`` -- The length of the content in bytes. 
    9192 
     93    If streaming file uploads are enabled two additional keys 
     94    describing the uploaded file will be present: 
     95 
     96        * ``tmpfilename`` -- The filename for the temporary file. 
     97        * ``tmpfile`` -- An open file object for the temporary file. 
     98 
     99    The temporary file will be removed when the request finishes. 
     100 
     101    Note that accessing ``content`` when streaming uploads are enabled 
     102    will read the whole file into memory which may not be what you want. 
     103 
    92104    Note that ``FILES`` will only contain data if the request method was POST 
    93105    and the ``<form>`` that posted to the request had 
    94106    ``enctype="multipart/form-data"``. Otherwise, ``FILES`` will be a blank 
  • docs/settings.txt

     
    521521 
    522522.. _Testing Django Applications: ../testing/ 
    523523 
     524FILE_UPLOAD_DIR 
     525--------------- 
     526 
     527Default: ``None`` 
     528 
     529Path to a directory where temporary files should be written during 
     530file uploads. Leaving this as ``None`` will disable streaming file uploads, 
     531and cause all uploaded files to be stored (temporarily) in memory. 
     532 
    524533IGNORABLE_404_ENDS 
    525534------------------ 
    526535 
     
    888897 
    889898.. _site framework docs: ../sites/ 
    890899 
     900STREAMING_MIN_POST_SIZE 
     901----------------------- 
     902 
     903Default: 524288 (``512*1024``) 
     904 
     905An integer specifying the minimum number of bytes that has to be 
     906received (in a POST) for file upload streaming to take place. Any 
     907request smaller than this will be handled in memory.  
     908Note: ``FILE_UPLOAD_DIR`` has to be defined to enable streaming. 
     909 
    891910TEMPLATE_CONTEXT_PROCESSORS 
    892911--------------------------- 
    893912 
  • docs/forms.txt

     
    475475   new_data = request.POST.copy() 
    476476   new_data.update(request.FILES) 
    477477 
     478Streaming file uploads. 
     479----------------------- 
     480 
     481File uploads will be read into memory by default. This works fine for 
     482small to medium sized uploads (from 1MB to 100MB depending on your 
     483setup and usage). If you want to support larger uploads you can enable 
     484upload streaming where only a small part of the file will be in memory 
     485at any time. To do this you need to specify the ``FILE_UPLOAD_DIR`` 
     486setting (see the settings_ document for more details). 
     487 
     488See `request object`_ for more details about ``request.FILES`` objects 
     489with streaming file uploads enabled. 
     490 
    478491Validators 
    479492========== 
    480493 
     
    698711.. _`generic views`: ../generic_views/ 
    699712.. _`models API`: ../model-api/ 
    700713.. _settings: ../settings/ 
     714.. _request object: ../request_response/#httprequest-objects