Ticket #2070: 2070_revision7359.diff

File 2070_revision7359.diff, 76.9 KB (added by axiak, 7 years ago)

New diff...some style changes and new documentation.

  • django/http/multipartparser.py

     
     1"""
     2MultiPart parsing for file uploads.
     3
     4This object will take the file upload headers
     5and the file upload handler and chunk the upload
     6data for the handler to deal with.
     7"""
     8from django.utils.datastructures import MultiValueDict
     9from django.utils.encoding import force_unicode
     10
     11__all__ = ('MultiPartParser','MultiPartParserError','InputStreamExhausted')
     12
     13class MultiPartParserError(Exception):
     14    pass
     15
     16class InputStreamExhausted(Exception):
     17    """ No more reads are allowed from this device. """
     18    pass
     19
     20class MultiPartParser(object):
     21    """
     22    A rfc2388 multipart/form-data parser.
     23
     24    parse() reads the input stream in chunk_size chunks and returns a
     25    tuple of (POST MultiValueDict, FILES MultiValueDict). If
     26    file_upload_dir is defined files will be streamed to temporary
     27    files in the specified directory.
     28    """
     29    def __init__(self, META, input_data, upload_handlers, encoding=None):
     30        """
     31        Initialize the MultiPartParser object.
     32
     33        *META* -- The standard META dictionary in Django request objects.
     34        *input_data* -- The raw post data, as a bytestring.
     35        *upload_handler* -- An object of type UploadHandler
     36                            that performs operations on the uploaded
     37                            data.
     38        *encoding* -- The encoding with which to treat the incoming data.
     39        """
     40        # Import cgi utilities for (near) future use.
     41        global parse_header, valid_boundary, settings
     42        from django.conf import settings
     43        from cgi import valid_boundary, parse_header
     44
     45        #
     46        # Content-Type should containt multipart and the boundary information.
     47        #
     48
     49        content_type = META.get('HTTP_CONTENT_TYPE', META.get('CONTENT_TYPE', ''))
     50        if not content_type.startswith('multipart/'):
     51            raise MultiPartParserError('Invalid Content-Type: %s' %
     52                                       content_type)
     53
     54        # Parse the header to get the boundary to split the parts.
     55        ctypes, opts = parse_header(content_type)
     56        boundary = opts.get('boundary')
     57        if not boundary or not valid_boundary(boundary):
     58            raise MultiPartParserError('Invalid boundary in multipart: %s' %
     59                                       boundary)
     60
     61
     62        #
     63        # Content-Length should contain the length of the body we are about
     64        # to receive.
     65        #
     66        try:
     67            content_length = int(META.get('HTTP_CONTENT_LENGTH',
     68                                          META.get('CONTENT_LENGTH',0)))
     69        except (ValueError, TypeError):
     70            # For now set it to 0...we'll try again later on down.
     71            content_length = 0
     72
     73        if content_length <= 0:
     74            # This means we shouldn't continue...raise an error.
     75            raise MultiPartParserError("Invalid content length: %r" % content_length)
     76
     77        self._boundary = boundary
     78        self._input_data = input_data
     79
     80        # For compatibility with low-level network APIs (with 32-bit integers),
     81        # the chunk size should be < 2^31, but still divisible by 4.
     82        self._chunk_size = min(2147483644, *[x.chunk_size for x in upload_handlers
     83                                            if x.chunk_size])
     84
     85        self._meta = META
     86        self._encoding = encoding or settings.DEFAULT_CHARSET
     87        self._content_length = content_length
     88        self._upload_handlers = upload_handlers
     89
     90    def parse(self):
     91        """
     92        Parse the POST data and break it into a FILES MultiValueDict
     93        and a POST MultiValueDict.
     94
     95           *returns* -- A tuple containing the POST and FILES dictionary,
     96                        respectively.
     97        """
     98        from django.core.files.fileuploadhandler import StopUpload, SkipFile
     99        from django.http import QueryDict
     100
     101        encoding = self._encoding
     102        handlers = self._upload_handlers
     103
     104        limited_input_data = LimitBytes(self._input_data, self._content_length)
     105
     106        # See if the handler will want to take care of the parsing.
     107        # This allows overriding everything if somebody wants it.
     108        for handler in handlers:
     109            result = handler.handle_raw_input(limited_input_data,
     110                                              self._meta,
     111                                              self._content_length,
     112                                              self._boundary,
     113                                              encoding)
     114            if result is not None:
     115                return result[0], result[1]
     116
     117        # Create the data structures to be used later.
     118        self._post = QueryDict('', mutable=True)
     119        self._files = MultiValueDict()
     120
     121        # Instantiate the parser and stream:
     122        stream = LazyStream(ChunkIter(limited_input_data, self._chunk_size))
     123
     124        # Whether or not to signal a file-completion at the beginning of the loop.
     125        old_field_name = None
     126        counters = [0] * len(handlers)
     127
     128        for item_type, meta_data, stream in Parser(stream, self._boundary):
     129            if old_field_name:
     130                # We run this at the beginning of the next loop
     131                # since we cannot be sure a file is complete until
     132                # we hit the next boundary/part of the multipart content.
     133                self.handle_file_complete(old_field_name, counters)
     134
     135            try:
     136                disposition = meta_data['content-disposition'][1]
     137                field_name = disposition['name'].strip()
     138            except (KeyError, IndexError, AttributeError):
     139                continue
     140
     141            transfer_encoding = meta_data.get('content-transfer-encoding')
     142            field_name = force_unicode(field_name, encoding, errors='replace')
     143
     144            if item_type == 'FIELD':
     145                # This is a post field, we can just set it in the post
     146                if transfer_encoding == 'base64':
     147                    raw_data = stream.read()
     148                    try:
     149                        data = str(raw_data).decode('base64')
     150                    except:
     151                        data = raw_data
     152                else:
     153                    data = stream.read()
     154
     155                self._post.appendlist(field_name,
     156                                      force_unicode(data, encoding, errors='replace'))
     157            elif item_type == 'FILE':
     158                # This is a file, use the handler...
     159                file_successful = True
     160                file_name = self.IE_sanitize(disposition.get('filename'))
     161                if not file_name:
     162                    continue
     163
     164                file_name = force_unicode(file_name, encoding, errors='replace')
     165
     166                content_type = meta_data.get('content-type', ('',))[0].strip()
     167                try:
     168                    charset = meta_data.get('content-type', (0,{}))[1].get('charset', None)
     169                except:
     170                    charset = None
     171
     172                try:
     173                    content_length = int(meta_data.get('content-length')[0])
     174                except (IndexError, TypeError, ValueError):
     175                    content_length = None
     176
     177                counters = [0] * len(handlers)
     178                try:
     179                    for handler in handlers:
     180                        retval = handler.new_file(field_name, file_name,
     181                                                  content_type, content_length,
     182                                                  charset)
     183                        if retval:
     184                            break
     185
     186                    for chunk in stream:
     187                        if transfer_encoding == 'base64':
     188                            # We only special-case base64 transfer encoding
     189                            try:
     190                                chunk = str(chunk).decode('base64')
     191                            except Exception, e:
     192                                # Since this is only a chunk, any error is an unfixable error.
     193                                raise MultiValueParseError("Could not decode base64 data: %r" % e)
     194
     195                        for i, handler in enumerate(handlers):
     196                            chunk_length = len(chunk)
     197                            counters[i] += chunk_length
     198                            chunk = handler.receive_data_chunk(chunk,
     199                                                               counters[i] - chunk_length,
     200                                                               counters[i])
     201                            if chunk is None:
     202                                # If the chunk received by the handler is None, then don't continue.
     203                                break
     204
     205                except (StopUpload, SkipFile), e:
     206                    file_successful = False
     207                    if isinstance(e, SkipFile):
     208                        # Just use up the rest of this file...
     209                        stream.exhaust()
     210                    elif isinstance(e, StopUpload):
     211                        # Abort the parsing and break
     212                        parser.abort()
     213                        break
     214                else:
     215                    # Handle file upload completions on next iteration.
     216                    old_field_name = field_name
     217            else:
     218                # If this is neither a FIELD or a FILE, just exhaust the stream.
     219                stream.exhuast()
     220
     221        # Make sure that the request data is all fed
     222        limited_input_data.exhaust()
     223
     224        # Signal that the upload has completed.
     225        for handler in handlers:
     226            retval = handler.upload_complete()
     227            if retval:
     228                break
     229
     230        return self._post, self._files
     231
     232    def handle_file_complete(self, old_field_name, counters):
     233        """
     234        Handle all the signalling that takes place when a file is complete.
     235        """
     236        for i, handler in enumerate(self._upload_handlers):
     237            file_obj = handler.file_complete(counters[i])
     238            if file_obj:
     239                # If it returns a file object, then set the files dict.
     240                self._files.appendlist(force_unicode(old_field_name,
     241                                                     self._encoding,
     242                                                     errors='replace'),
     243                                       file_obj)
     244                break
     245
     246    def IE_sanitize(self, filename):
     247        """cleanup filename from IE full paths"""
     248        return filename and filename[filename.rfind("\\")+1:].strip()
     249
     250
     251
     252class LazyStream(object):
     253    """
     254    The LazyStream wrapper allows one to pull and "unget" bytes from a stream.
     255
     256    Given a producer object (an iterator that yields bytestrings), the
     257    LazyStream object will support iteration, reading, and keeping a
     258    "look-back" variable in case you need to "unget" some bytes.
     259    """
     260    def __init__(self, producer, length=None):
     261        """
     262        Every LazyStream must have a producer when instantiated.
     263
     264        A producer is an iterable that returns a string each time it
     265        is called.
     266        """
     267        self._producer = producer
     268        self._empty = False
     269        self._leftover = ''
     270        self.length = length
     271        self.position = 0
     272        self._remaining = length
     273
     274    def tell(self):
     275        return self.position
     276
     277    def read(self, size=None):
     278        def parts():
     279            remaining = (size is not None and [size] or [self._remaining])[0]
     280            # do the whole thing in one shot if no limit was provided.
     281            if remaining is None:
     282                yield ''.join(self)
     283                return
     284
     285            # otherwise do some bookkeeping to return exactly enough
     286            # of the stream and stashing any extra content we get from
     287            # the producer
     288            while remaining != 0:
     289                assert remaining > 0, 'remaining bytes to read should never go negative'
     290
     291                chunk = self.next()
     292
     293                emitting = chunk[:remaining]
     294                self.unget(chunk[remaining:])
     295                remaining -= len(emitting)
     296                yield emitting
     297
     298        out = ''.join(parts())
     299        self.position += len(out)
     300        return out
     301
     302    def next(self):
     303        """
     304        Used when the exact number of bytes to read is unimportant.
     305
     306        This procedure just returns whatever is chunk is conveniently
     307        returned from the iterator instead. Useful to avoid
     308        unnecessary bookkeeping if performance is an issue.
     309        """
     310        if self._leftover:
     311            output = self._leftover
     312            self.position += len(output)
     313            self._leftover = ''
     314            return output
     315        else:
     316            output = self._producer.next()
     317            self.position += len(output)
     318            return output
     319
     320    def close(self):
     321        """
     322        Used to invalidate/disable this lazy stream.
     323
     324        Replaces the producer with an empty list. Any leftover bytes
     325        that have already been read will still be reported upon read()
     326        and/or next().
     327        """
     328        self._producer = []
     329
     330    def __iter__(self):
     331        return self
     332
     333    def unget(self, bytes):
     334        """
     335        Places bytes back onto the front of the lazy stream.
     336
     337        Future calls to read() will return those bytes first. The
     338        stream position and thus tell() will be rewound.
     339        """
     340        self.position -= len(bytes)
     341        self._leftover = ''.join([bytes, self._leftover])
     342
     343    def exhaust(self):
     344        """
     345        Exhausts the entire underlying stream.
     346
     347        Useful for skipping and advancing sections.
     348        """
     349        for thing in self:
     350            pass
     351
     352
     353class ChunkIter(object):
     354    """
     355    An iterable that will yield chunks of data.
     356    Given a file-like object as the constructor,
     357    this object will yield chunks of read operations
     358    from that object.
     359    """
     360    def __init__(self, flo, chunk_size=64 * 1024):
     361        self.flo = flo
     362        self.chunk_size = chunk_size
     363
     364    def next(self):
     365        try:
     366            data = self.flo.read(self.chunk_size)
     367        except InputStreamExhausted:
     368            raise StopIteration
     369        if data:
     370            return data
     371        else:
     372            raise StopIteration
     373
     374    def __iter__(self):
     375        return self
     376
     377
     378class LimitBytes(object):
     379    """ Limit bytes for a file object. """
     380    def __init__(self, fileobject, length):
     381        self._file = fileobject
     382        self.remaining = length
     383
     384    def read(self, num_bytes=None):
     385        """
     386        Read data from the underlying file.
     387        If you ask for too much or there isn't anything left,
     388        this will raise an InputStreamExhausted error.
     389        """
     390        if self.remaining <= 0:
     391            raise InputStreamExhausted()
     392        if num_bytes is None:
     393            num_bytes = self.remaining
     394        else:
     395            num_bytes = min(num_bytes, self.remaining)
     396        self.remaining -= num_bytes
     397        return self._file.read(num_bytes)
     398
     399    def exhaust(self):
     400        """
     401        Exhaust this file until all of the bytes it was limited by
     402        have been read.
     403        """
     404        while self.remaining > 0:
     405            num_bytes = min(self.remaining, 16384)
     406            __ = self._file.read(num_bytes)
     407            self.remaining -= num_bytes
     408
     409
     410class InterBoundaryIter(object):
     411    """
     412    A Producer that will iterate over boundaries.
     413    """
     414    def __init__(self, stream, boundary):
     415        self._stream = stream
     416        self._boundary = boundary
     417
     418    def __iter__(self):
     419        return self
     420
     421    def next(self):
     422        try:
     423            return LazyStream(BoundaryIter(self._stream, self._boundary))
     424        except InputStreamExhausted:
     425            raise StopIteration
     426
     427
     428class BoundaryIter(object):
     429    """
     430    A Producer that is sensitive to boundaries.
     431
     432    Will happily yield bytes until a boundary is found. Will yield the
     433    bytes before the boundary, throw away the boundary bytes
     434    themselves, and push the post-boundary bytes back on the stream.
     435
     436    The future calls to .next() after locating the boundary will raise
     437    a StopIteration exception.
     438    """
     439
     440    def __init__(self, stream, boundary):
     441        self._stream = stream
     442        self._boundary = boundary
     443        self._done = False
     444        # rollback an additional six bytes because the format is like
     445        # this: CRLF<boundary>[--CRLF]
     446        self._rollback = len(boundary) + 6
     447
     448        # Try to use mx fast string search if available. Otherwise
     449        # use Python find. Wrap the latter for consistency.
     450        unused_char = self._stream.read(1)
     451        if not unused_char:
     452            raise InputStreamExhausted
     453        self._stream.unget(unused_char)
     454        try:
     455            from mx.TextTools import FS
     456            self._fs = FS(boundary).find
     457        except ImportError:
     458            self._fs = lambda data: data.find(boundary)
     459
     460    def __iter__(self):
     461        return self
     462
     463    def next(self):
     464        if self._done:
     465            raise StopIteration
     466
     467        stream = self._stream
     468        rollback = self._rollback
     469
     470        bytes_read = 0
     471        chunks = []
     472        for bytes in stream:
     473            bytes_read += len(bytes)
     474            chunks.append(bytes)
     475            if bytes_read > rollback:
     476                break
     477            if not bytes:
     478                break
     479        else:
     480            self._done = True
     481
     482        if not chunks:
     483            raise StopIteration
     484
     485        chunk = ''.join(chunks)
     486        boundary = self._find_boundary(chunk, len(chunk) < self._rollback)
     487
     488        if boundary:
     489            end, next = boundary
     490            stream.unget(chunk[next:])
     491            self._done = True
     492            return chunk[:end]
     493        else:
     494            # make sure we dont treat a partial boundary (and
     495            # its separators) as data
     496            if not chunk[:-rollback]:# and len(chunk) >= (len(self._boundary) + 6):
     497                # There's nothing left, we should just return and mark as done.
     498                self._done = True
     499                return chunk
     500            else:
     501                stream.unget(chunk[-rollback:])
     502                return chunk[:-rollback]
     503
     504    def _find_boundary(self, data, eof = False):
     505        """
     506        Finds a multipart boundary in data.
     507
     508        Should no boundry exist in the data None is returned
     509        instead. Otherwise a tuple containing
     510        the indices of the following are returned:
     511
     512         * the end of current encapsulation
     513
     514         * the start of the next encapsulation
     515        """
     516        index = self._fs(data)
     517        if index < 0:
     518            return None
     519        else:
     520            end = index
     521            next = index + len(self._boundary)
     522            data_len = len(data) - 1
     523            # backup over CRLF
     524            if data[max(0,end-1)] == '\n': end -= 1
     525            if data[max(0,end-1)] == '\r': end -= 1
     526            # skip over --CRLF
     527            if data[min(data_len,next)] == '-': next += 1
     528            if data[min(data_len,next)] == '-': next += 1
     529            if data[min(data_len,next)] == '\r': next += 1
     530            if data[min(data_len,next)] == '\n': next += 1
     531            return end, next
     532
     533
     534def ParseBoundaryStream(stream, max_header_size):
     535        """
     536        Parses one and exactly one stream that encapsulates a boundary.
     537        """
     538        # Stream at beginning of header, look for end of header
     539        # and parse it if found. The header must fit within one
     540        # chunk.
     541        chunk = stream.read(max_header_size)
     542        # 'find' returns the top of these four bytes, so we'll
     543        # need to munch them later to prevent them from polluting
     544        # the payload.
     545        header_end = chunk.find('\r\n\r\n')
     546
     547        def parse_header(line):
     548            from cgi import parse_header
     549            main_value_pair, params = parse_header(line)
     550            try:
     551                name, value = main_value_pair.split(':', 1)
     552            except:
     553                raise ValueError("Invalid header: %r" % line)
     554            return name, (value, params)
     555
     556        if header_end == -1:
     557            # we find no header, so we just mark this fact and pass on
     558            # the stream verbatim
     559            stream.unget(chunk)
     560            return ('RAW', {}, stream)
     561
     562        header = chunk[:header_end]
     563
     564        # here we place any excess chunk back onto the stream, as
     565        # well as throwing away the CRLFCRLF bytes from above.
     566        stream.unget(chunk[header_end + 4:])
     567
     568        is_file_field = False
     569        outdict = {}
     570
     571        # Eliminate blank lines
     572        for line in header.split('\r\n'):
     573            # This terminology ("main value" and "dictionary of
     574            # parameters") is from the Python docs.
     575            name, (value, params) = parse_header(line)
     576            if name == 'content-disposition' and params.get('filename'):
     577                is_file_field = True
     578
     579            outdict[name] = value, params
     580
     581        if is_file_field:
     582            return ('FILE', outdict, stream)
     583        else:
     584            return ('FIELD', outdict, stream)
     585
     586
     587class Parser(object):
     588    def __init__(self, stream, boundary):
     589        self._stream = stream
     590        self._separator = '--' + boundary
     591
     592    def __iter__(self):
     593
     594        boundarystream = InterBoundaryIter(self._stream,
     595                                           self._separator)
     596
     597        for sub_stream in boundarystream:
     598            # Iterate over each part
     599            yield ParseBoundaryStream(sub_stream, 1024)
  • django/http/__init__.py

     
    1111
    1212from django.utils.datastructures import MultiValueDict, FileDict
    1313from django.utils.encoding import smart_str, iri_to_uri, force_unicode
    14 
     14from django.http.multipartparser import MultiPartParser
    1515from utils import *
    1616
    1717RESERVED_CHARS="!*'();:@&=+$,/?%#[]"
     
    3030        self.GET, self.POST, self.COOKIES, self.META, self.FILES = {}, {}, {}, {}, {}
    3131        self.path = ''
    3232        self.method = None
     33        self._upload_handlers = []
    3334
    3435    def __repr__(self):
    3536        return '<HttpRequest\nGET:%s,\nPOST:%s,\nCOOKIES:%s,\nMETA:%s>' % \
     
    102103
    103104    encoding = property(_get_encoding, _set_encoding)
    104105
    105 def parse_file_upload(header_dict, post_data):
    106     """Returns a tuple of (POST QueryDict, FILES MultiValueDict)."""
    107     import email, email.Message
    108     from cgi import parse_header
    109     raw_message = '\r\n'.join(['%s:%s' % pair for pair in header_dict.items()])
    110     raw_message += '\r\n\r\n' + post_data
    111     msg = email.message_from_string(raw_message)
    112     POST = QueryDict('', mutable=True)
    113     FILES = MultiValueDict()
    114     for submessage in msg.get_payload():
    115         if submessage and isinstance(submessage, email.Message.Message):
    116             name_dict = parse_header(submessage['Content-Disposition'])[1]
    117             # name_dict is something like {'name': 'file', 'filename': 'test.txt'} for file uploads
    118             # or {'name': 'blah'} for POST fields
    119             # We assume all uploaded files have a 'filename' set.
    120             if 'filename' in name_dict:
    121                 assert type([]) != type(submessage.get_payload()), "Nested MIME messages are not supported"
    122                 if not name_dict['filename'].strip():
    123                     continue
    124                 # IE submits the full path, so trim everything but the basename.
    125                 # (We can't use os.path.basename because that uses the server's
    126                 # directory separator, which may not be the same as the
    127                 # client's one.)
    128                 filename = name_dict['filename'][name_dict['filename'].rfind("\\")+1:]
    129                 FILES.appendlist(name_dict['name'], FileDict({
    130                     'filename': filename,
    131                     'content-type': 'Content-Type' in submessage and submessage['Content-Type'] or None,
    132                     'content': submessage.get_payload(),
    133                 }))
    134             else:
    135                 POST.appendlist(name_dict['name'], submessage.get_payload())
    136     return POST, FILES
     106    def _set_upload_handlers(self, upload_handlers):
     107        """
     108        Set the upload handler to the new handler given in the parameter.
     109        """
     110        if hasattr(self, '_files'):
     111            raise AttributeError("You cannot set the upload handler after the upload has been processed.")
     112        self._upload_handlers = upload_handlers
    137113
     114    def _get_upload_handlers(self):
     115        return self._upload_handlers
    138116
     117    upload_handlers = property(_get_upload_handlers, _set_upload_handlers)
     118
     119    def parse_file_upload(self, META, post_data):
     120        """Returns a tuple of (POST QueryDict, FILES MultiValueDict)."""
     121        from django.core.files.fileuploadhandler import TemporaryFileUploadHandler, MemoryFileUploadHandler
     122        if not self.upload_handlers:
     123            # The order of the upload handlers specifies the order in which
     124            # the handlers receive data. One handler may or may not
     125            # prevent the execution of subsequent handlers.
     126            self.upload_handlers = (MemoryFileUploadHandler(),
     127                                    TemporaryFileUploadHandler())
     128        else:
     129            # By using tuple(), we render the upload_handlers
     130            # data structure immutable.
     131            self.upload_handlers = tuple(self.upload_handlers)
     132
     133        parser = MultiPartParser(META, post_data, self.upload_handlers,
     134                                 self.encoding)
     135        return parser.parse()
     136
     137
    139138class QueryDict(MultiValueDict):
    140139    """
    141140    A specialized MultiValueDict that takes a query string when initialized.
  • django/test/client.py

     
    1818BOUNDARY = 'BoUnDaRyStRiNg'
    1919MULTIPART_CONTENT = 'multipart/form-data; boundary=%s' % BOUNDARY
    2020
     21class FakePayload(object):
     22    """
     23    A wrapper around StringIO that restricts what can be read,
     24    since data from the network can't be seeked and cannot
     25    be read outside of its content length (or else we hang).
     26    """
     27    def __init__(self, content):
     28        self.__content = StringIO(content)
     29        self.__len = len(content)
     30
     31    def read(self, num_bytes=None):
     32        if num_bytes is None:
     33            num_bytes = self.__len or 1
     34        assert self.__len >= num_bytes, "Cannot read more than the available bytes from the HTTP incoming data."
     35        content = self.__content.read(num_bytes)
     36        self.__len -= num_bytes
     37        return content
     38
     39
    2140class ClientHandler(BaseHandler):
    2241    """
    2342    A HTTP Handler that can be used for testing purposes.
     
    230249            'CONTENT_TYPE':   content_type,
    231250            'PATH_INFO':      urllib.unquote(path),
    232251            'REQUEST_METHOD': 'POST',
    233             'wsgi.input':     StringIO(post_data),
     252            'wsgi.input':     FakePayload(post_data),
    234253        }
    235254        r.update(extra)
    236255
  • django/conf/global_settings.py

     
    224224# Example: "http://media.lawrence.com"
    225225MEDIA_URL = ''
    226226
     227# Directory to upload streamed files temporarily.
     228# A value of `None` means that it will use the default temporary
     229# directory for the server's operating system.
     230FILE_UPLOAD_TEMP_DIR = None
     231
    227232# Default formatting for date objects. See all available format strings here:
    228233# http://www.djangoproject.com/documentation/templates/#now
    229234DATE_FORMAT = 'N j, Y'
  • django/db/models/base.py

     
    1313from django.utils.datastructures import SortedDict
    1414from django.utils.functional import curry
    1515from django.utils.encoding import smart_str, force_unicode, smart_unicode
     16from django.core.files.filemove import file_move_safe
    1617from django.conf import settings
    1718from itertools import izip
    1819import types
     
    384385    def _get_FIELD_size(self, field):
    385386        return os.path.getsize(self._get_FIELD_filename(field))
    386387
    387     def _save_FIELD_file(self, field, filename, raw_contents, save=True):
     388    def _save_FIELD_file(self, field, filename, raw_field, save=True):
    388389        directory = field.get_directory_name()
    389390        try: # Create the date-based directory if it doesn't exist.
    390391            os.makedirs(os.path.join(settings.MEDIA_ROOT, directory))
    391392        except OSError: # Directory probably already exists.
    392393            pass
     394
     395        if filename is None:
     396            filename = raw_field.file_name
     397
    393398        filename = field.get_filename(filename)
    394399
    395400        # If the filename already exists, keep adding an underscore to the name of
     
    406411        setattr(self, field.attname, filename)
    407412
    408413        full_filename = self._get_FIELD_filename(field)
    409         fp = open(full_filename, 'wb')
    410         fp.write(raw_contents)
    411         fp.close()
     414        if hasattr(raw_field, 'temporary_file_path'):
     415            raw_field.close()
     416            file_move_safe(raw_field.temporary_file_path(), full_filename)
     417        else:
     418            from django.core.files import filelocks
     419            fp = open(full_filename, 'wb')
     420            # exclusive lock
     421            filelocks.lock(fp, filelocks.LOCK_EX)
     422            # Stream it into the file, from where it is.
     423            for chunk in raw_field.chunk(65535):
     424                fp.write(chunk)
     425            fp.close()
    412426
    413427        # Save the width and/or height, if applicable.
    414428        if isinstance(field, ImageField) and (field.width_field or field.height_field):
  • django/db/models/fields/__init__.py

     
    785785        setattr(cls, 'get_%s_filename' % self.name, curry(cls._get_FIELD_filename, field=self))
    786786        setattr(cls, 'get_%s_url' % self.name, curry(cls._get_FIELD_url, field=self))
    787787        setattr(cls, 'get_%s_size' % self.name, curry(cls._get_FIELD_size, field=self))
    788         setattr(cls, 'save_%s_file' % self.name, lambda instance, filename, raw_contents, save=True: instance._save_FIELD_file(self, filename, raw_contents, save))
     788        setattr(cls, 'save_%s_file' % self.name, lambda instance, filename, raw_field, save=True: instance._save_FIELD_file(self, filename, raw_field, save))
     789        setattr(cls, 'move_%s_file' % self.name, lambda instance, raw_field, save=True: instance._save_FIELD_file(self, None, raw_field, save))
    789790        dispatcher.connect(self.delete_file, signal=signals.post_delete, sender=cls)
    790791
    791792    def delete_file(self, instance):
     
    808809        if new_data.get(upload_field_name, False):
    809810            func = getattr(new_object, 'save_%s_file' % self.name)
    810811            if rel:
    811                 func(new_data[upload_field_name][0]["filename"], new_data[upload_field_name][0]["content"], save)
     812                func(new_data[upload_field_name][0].file_name, new_data[upload_field_name][0], save)
    812813            else:
    813                 func(new_data[upload_field_name]["filename"], new_data[upload_field_name]["content"], save)
     814                func(new_data[upload_field_name].file_name, new_data[upload_field_name], save)
    814815
    815816    def get_directory_name(self):
    816817        return os.path.normpath(force_unicode(datetime.datetime.now().strftime(smart_str(self.upload_to))))
     
    823824    def save_form_data(self, instance, data):
    824825        from django.newforms.fields import UploadedFile
    825826        if data and isinstance(data, UploadedFile):
    826             getattr(instance, "save_%s_file" % self.name)(data.filename, data.content, save=False)
     827            getattr(instance, "save_%s_file" % self.name)(data.filename, data.data, save=False)
    827828
    828829    def formfield(self, **kwargs):
    829830        defaults = {'form_class': forms.FileField}
  • django/oldforms/__init__.py

     
    680680        self.field_name, self.is_required = field_name, is_required
    681681        self.validator_list = [self.isNonEmptyFile] + validator_list
    682682
    683     def isNonEmptyFile(self, field_data, all_data):
    684         try:
    685             content = field_data['content']
    686         except TypeError:
    687             raise validators.CriticalValidationError, ugettext("No file was submitted. Check the encoding type on the form.")
    688         if not content:
     683    def isNonEmptyFile(self, new_data, all_data):
     684        if hasattr(new_data, 'upload_errors'):
     685            upload_errors = new_data.upload_errors()
     686            if upload_errors:
     687                raise validators.CriticalValidationError, upload_errors
     688        if not new_data.file_size:
    689689            raise validators.CriticalValidationError, ugettext("The submitted file is empty.")
    690690
    691691    def render(self, data):
    692692        return mark_safe(u'<input type="file" id="%s" class="v%s" name="%s" />' % \
    693693            (self.get_id(), self.__class__.__name__, self.field_name))
    694694
     695    def prepare(self, new_data):
     696        if hasattr(new_data, 'upload_errors'):
     697            upload_errors = new_data.upload_errors()
     698            new_data[self.field_name] = { '_file_upload_error': upload_errors }
     699
    695700    def html2python(data):
    696701        if data is None:
    697702            raise EmptyValue
  • django/core/handlers/wsgi.py

     
    7878        self.path = force_unicode(environ['PATH_INFO'])
    7979        self.META = environ
    8080        self.method = environ['REQUEST_METHOD'].upper()
     81        self._upload_handlers = []
    8182
    8283    def __repr__(self):
    8384        # Since this is called as part of error handling, we need to be very
     
    112113        # Populates self._post and self._files
    113114        if self.method == 'POST':
    114115            if self.environ.get('CONTENT_TYPE', '').startswith('multipart'):
    115                 header_dict = dict([(k, v) for k, v in self.environ.items() if k.startswith('HTTP_')])
    116                 header_dict['Content-Type'] = self.environ.get('CONTENT_TYPE', '')
    117                 self._post, self._files = http.parse_file_upload(header_dict, self.raw_post_data)
     116                self._raw_post_data = ''
     117                self._post, self._files = self.parse_file_upload(self.META, self.environ['wsgi.input'])
    118118            else:
    119119                self._post, self._files = http.QueryDict(self.raw_post_data, encoding=self._encoding), datastructures.MultiValueDict()
    120120        else:
  • django/core/handlers/modpython.py

     
    1616    def __init__(self, req):
    1717        self._req = req
    1818        self.path = force_unicode(req.uri)
     19        self._upload_handlers = []
    1920
    2021    def __repr__(self):
    2122        # Since this is called as part of error handling, we need to be very
     
    5354    def _load_post_and_files(self):
    5455        "Populates self._post and self._files"
    5556        if 'content-type' in self._req.headers_in and self._req.headers_in['content-type'].startswith('multipart'):
    56             self._post, self._files = http.parse_file_upload(self._req.headers_in, self.raw_post_data)
     57            self._raw_post_data = ''
     58            self._post, self._files = self.parse_file_upload(self.META, self._req)
    5759        else:
    5860            self._post, self._files = http.QueryDict(self.raw_post_data, encoding=self._encoding), datastructures.MultiValueDict()
    5961
  • django/core/files/filelocks.py

     
     1"""
     2Locking portability based partially on example by
     3Jonathan Feignberg <jdf@pobox.com> in python cookbook.
     4
     5Example Usage::
     6
     7    from django.utils import file_locks
     8
     9    f = open('./file', 'wb')
     10
     11    file_locks.lock(f, file_locks.LOCK_EX)
     12    f.write('Django')
     13    f.close()
     14"""
     15
     16__all__ = ('LOCK_EX','LOCK_SH','LOCK_NB','lock','unlock')
     17
     18system_type = None
     19
     20try:
     21        import win32con
     22        import win32file
     23        import pywintypes
     24        LOCK_EX = win32con.LOCKFILE_EXCLUSIVE_LOCK
     25        LOCK_SH = 0
     26        LOCK_NB = win32con.LOCKFILE_FAIL_IMMEDIATELY
     27        __overlapped = pywintypes.OVERLAPPED()
     28        system_type = 'nt'
     29except (ImportError, AttributeError):
     30        pass
     31
     32try:
     33        import fcntl
     34        LOCK_EX = fcntl.LOCK_EX
     35        LOCK_SH = fcntl.LOCK_SH
     36        LOCK_NB = fcntl.LOCK_NB
     37        system_type = 'posix'
     38except (ImportError, AttributeError):
     39        pass
     40
     41
     42
     43if system_type == 'nt':
     44        def lock(file, flags):
     45                hfile = win32file._get_osfhandle(file.fileno())
     46                win32file.LockFileEx(hfile, flags, 0, -0x10000, __overlapped)
     47
     48        def unlock(file):
     49                hfile = win32file._get_osfhandle(file.fileno())
     50                win32file.UnlockFileEx(hfile, 0, -0x10000, __overlapped)
     51
     52elif system_type =='posix':
     53        def lock(file, flags):
     54                fcntl.flock(file.fileno(), flags)
     55
     56        def unlock(file):
     57                fcntl.flock(file.fileno(), fcntl.LOCK_UN)
     58
     59else:
     60        # File locking is not supported.
     61        LOCK_EX = LOCK_SH = LOCK_NB = None
     62       
     63        # Dummy functions that don't do anything.
     64        def lock(file, flags):
     65                pass
     66
     67        def unlock(file):
     68                pass
  • django/core/files/uploadedfile.py

     
     1"""
     2The uploaded file objects for Django.
     3This contains the base UploadedFile and the TemporaryUploadedFile
     4derived class.
     5"""
     6
     7__all__ = ('UploadedFile', 'TemporaryUploadedFile', 'InMemoryUploadedFile')
     8
     9class UploadedFile(object):
     10    """
     11    The UploadedFile object behaves somewhat like a file
     12    object and represents some data that the user submitted
     13    and is stored in some form.
     14    """
     15    DEFAULT_CHUNK_SIZE = 64 * 2**10
     16
     17    def __init__(self):
     18        self.file_size = None
     19        self.file_name = None
     20        self.content_type = None
     21        self.charset = None
     22        pass
     23
     24    def file_size(self):
     25        return self.file_size
     26
     27    def chunk(self, chunk_size=None):
     28        """
     29        Read the file to generate chunks of chunk_size bytes.
     30        """
     31        if not chunk_size:
     32            chunk_size = UploadedFile.DEFAULT_CHUNK_SIZE
     33
     34        if hasattr(self, 'seek'):
     35            self.seek(0)
     36        # Assume the pointer is at zero...
     37        counter = self.file_size()
     38
     39        while counter > 0:
     40            yield self.read(chunk_size)
     41            counter -= chunk_size
     42
     43
     44    def multiple_chunks(self, chunk_size=None):
     45        """
     46        Return True if you can expect multiple chunks, False otherwise.
     47        Note: If a particular file representation is in memory, then
     48              override this to return False.
     49        """
     50        if not chunk_size:
     51            chunk_size = UploadedFile.DEFAULT_CHUNK_SIZE
     52        return self.file_size() < chunk_size
     53       
     54
     55    def read(self, num_bytes=None):
     56        """
     57        Read from the file in whatever representation it has.
     58        """
     59        raise NotImplementedError()
     60
     61    def open(self):
     62        """
     63        Open the file, if one needs to.
     64        """
     65        pass
     66
     67
     68    def close(self):
     69        """
     70        Close the file, if one needs to.
     71        """
     72        pass
     73
     74    def __getitem__(self, key):
     75        """
     76        This maintains backwards compatibility.
     77        """
     78        import warnings
     79        warnings.warn("The dictionary access of uploaded file objects is deprecated. Use the new object interface instead.", DeprecationWarning)
     80        # Dictionary to translate labels
     81        # for backwards compatbility.
     82        # Should be removed at some point.
     83        backwards_translate = {
     84            'filename': 'file_name',
     85            'content-type': 'content_type',
     86            }
     87
     88        if key == 'content':
     89            return self.read()
     90        else:
     91            return getattr(self, backwards_translate.get(key, key))
     92
     93    def __repr__(self):
     94        """
     95        This representation could be anything and can be overridden.
     96        This is mostly done to make it look somewhat useful.
     97        """
     98        _dict = {
     99            'file_name': self.file_name,
     100            'content_type': self.content_type,
     101            'content': '<omitted>',
     102            }
     103        return repr(_dict)
     104
     105
     106class TemporaryUploadedFile(UploadedFile):
     107    """
     108    Upload a file to a temporary file.
     109    """
     110
     111    def __init__(self, file, file_name, content_type, file_size, charset):
     112        self.file = file
     113        self.file_name = file_name
     114        self.path = file.name
     115        self.content_type = content_type
     116        self.file_size = file_size
     117        self.charset = charset
     118        self.file.seek(0)
     119
     120    def temporary_file_path(self):
     121        """
     122        Return the full path of this file.
     123        """
     124        return self.path
     125
     126    def read(self, *args, **kwargs):
     127        return self.file.read(*args, **kwargs)
     128
     129    def open(self):
     130        """
     131        Assume the person meant to seek.
     132        """
     133        self.seek(0)
     134
     135    def seek(self, *args, **kwargs):
     136        self.file.seek(*args, **kwargs)
     137
     138
     139class InMemoryUploadedFile(UploadedFile):
     140    """
     141    Upload a file into memory.
     142    """
     143    def __init__(self, file, field_name, file_name, content_type, charset):
     144        self.file = file
     145        self.field_name = field_name
     146        self.file_name = file_name
     147        self.content_type = content_type
     148        self.charset = charset
     149        self.file.seek(0)
     150
     151    def seek(self, *args, **kwargs):
     152        self.file.seek(*args, **kwargs)
     153
     154    def open(self):
     155        self.seek(0)
     156
     157    def read(self, *args, **kwargs):
     158        return self.file.read(*args, **kwargs)
     159
     160    def chunk(self, chunk_size=None):
     161        """
     162        Return the entirety of the data regardless.
     163        """
     164        self.file.seek(0)
     165        return self.read()
     166
     167    def multiple_chunks(self, chunk_size=None):
     168        """
     169        Since it's in memory, we'll never have multiple chunks.
     170        """
     171        return False
     172
     173
     174class SimpleUploadedFile(InMemoryUploadedFile):
     175    """
     176    A simple representation of a file, which
     177    just has content, size, and a name.
     178    """
     179    def __init__(self, name, content, content_type='text/plain'):
     180        try:
     181            from cStringIO import StringIO
     182        except ImportError:
     183            from StringIO import StringIO
     184        self.file = StringIO(content or '')
     185        self.file_name = name
     186        self.field_name = None
     187        self.file_size = len(content or '')
     188        self.content_type = content_type
     189        self.charset = None
     190        self.file.seek(0)
     191
  • django/core/files/__init__.py

     
     1
  • django/core/files/fileuploadhandler.py

     
     1""" A fileuploadhandler base and default subclass for handling file uploads.
     2"""
     3import os
     4try:
     5    from cStringIO import StringIO
     6except ImportError:
     7    from StringIO import StringIO
     8
     9from django.utils.encoding import force_unicode
     10from django.utils.datastructures import MultiValueDict
     11
     12from django.core.files.uploadedfile import TemporaryUploadedFile, InMemoryUploadedFile
     13
     14__all__ = ('UploadFileException','StopUpload', 'SkipFile',
     15           'FileUploadHandler', 'TemporaryFileUploadHandler',
     16           'MemoryFileUploadHandler')
     17
     18
     19class UploadFileException(Exception):
     20    """ Any error having to do with Uploading Files. """
     21    pass
     22
     23class StopUpload(UploadFileException):
     24    """ This exception is raised when an upload must abort. """
     25    pass
     26
     27class SkipFile(UploadFileException):
     28    """ This exception is raised when a file needs to be skipped. """
     29    pass
     30
     31
     32class FileUploadHandler(object):
     33    """ FileUploadHandler will take data and handle file uploads
     34    in a streamed fashion.
     35    """
     36    chunk_size = 64 * 2 ** 10 #: The default chunk size is 64 KB.
     37
     38    def __init__(self):
     39        " Initialize some local variables. "
     40        self.file_name = None
     41        self.content_type = None
     42        self.content_length = None
     43        self.charset = None
     44
     45    def handle_raw_input(self, input_data, META, content_length, boundary, encoding=None):
     46        """
     47        Handle the raw input from the client.
     48        Parameters:
     49          *input_data* -- An object that supports reading via .read().
     50          *content_length* -- The (integer) value of the Content-Length header from the client.
     51          *boundary* -- The boundary from the Content-Type header. Be sure to prepend two '--'.
     52        """
     53        pass
     54
     55    def new_file(self, field_name, file_name, content_type, content_length, charset=None):
     56        """
     57        Signal that a new file has been started.
     58       
     59        Warning: Do not trust content_length, if you get it at all.
     60        """
     61        self.field_name = field_name
     62        self.file_name = file_name
     63        self.content_type = content_type
     64        self.content_length = content_length
     65        self.charset = charset
     66
     67    def receive_data_chunk(self, raw_data, start, stop):
     68        """
     69        Receive data from the streamed upload parser.
     70        Start and stop are the positions in the file.
     71        This equality should always be true::
     72            len(raw_data) = stop - start
     73        """
     74        raise NotImplementedError()
     75
     76    def file_complete(self, file_size):
     77        """
     78        Signal that a file has completed.
     79        File size corresponds to the actual size accumulated
     80        by all the chunks.
     81
     82        This should return a valid UploadedFile object.
     83        """
     84        raise NotImplementedError()
     85
     86    def upload_complete(self):
     87        """
     88        Signal that the upload is complete.
     89        Do any cleanup that is necessary for this handler.
     90        """
     91        pass
     92
     93
     94
     95class TemporaryFileUploadHandler(FileUploadHandler):
     96    """
     97    Upload the streaming data into a temporary file.
     98    """
     99    def __init__(self, *args, **kwargs):
     100        """ Import settings for later. """
     101        super(TemporaryFileUploadHandler, self).__init__(*args, **kwargs)
     102        global settings
     103        from django.conf import settings
     104
     105    def new_file(self, file_name, *args, **kwargs):
     106        """
     107        Create the file object to append to as data is coming in.
     108        """
     109        super(TemporaryFileUploadHandler, self).new_file(file_name, *args, **kwargs)
     110        self.file = TemporaryFile(settings.FILE_UPLOAD_TEMP_DIR)
     111        self.write = self.file.write
     112
     113    def receive_data_chunk(self, raw_data, start, stop):
     114        """
     115        Once we get the data, we will save it to our file.
     116        """
     117        self.write(raw_data)
     118
     119    def file_complete(self, file_size):
     120        """
     121        Signal that a file has completed.
     122        File size corresponds to the actual size accumulated
     123        by all the chunks.
     124
     125        This should return a valid UploadedFile object.
     126        """
     127        self.file.seek(0)
     128        return TemporaryUploadedFile(self.file, self.file_name,
     129                                     self.content_type, file_size,
     130                                     self.charset)
     131
     132
     133class TemporaryFile(object):
     134    """
     135    A temporary file that tries to delete itself when garbage collected.
     136    """
     137    def __init__(self, dir):
     138        import tempfile
     139        if not dir:
     140            dir = tempfile.gettempdir()
     141        try:
     142            (fd, name) = tempfile.mkstemp(suffix='.upload', dir=dir)
     143            self.file = os.fdopen(fd, 'w+b')
     144        except (OSError, IOError):
     145            raise OSError, "Could not create temporary file for uploading, have you set settings.FILE_UPLOAD_TEMP_DIR correctly?"
     146        self.name = name
     147
     148    def __getattr__(self, name):
     149        a = getattr(self.__dict__['file'], name)
     150        if type(a) != type(0):
     151            setattr(self, name, a)
     152        return a
     153
     154    def __del__(self):
     155        try:
     156            os.unlink(self.name)
     157        except OSError:
     158            pass
     159
     160
     161class MemoryFileUploadHandler(FileUploadHandler):
     162    """
     163    The MemoryFileUploadHandler will place the data directly into memory.
     164    """
     165
     166    def __init__(self):
     167        pass
     168
     169
     170    def handle_raw_input(self, input_data, META, content_length, boundary, encoding=None):
     171        """
     172        Parse the input data in-memory.
     173        """
     174        if content_length > 2621440:
     175            # If the post is greater than 2.5 MB, do nothing.
     176            return
     177
     178        from django.http import QueryDict
     179        import email, email.Message
     180        from cgi import parse_header
     181
     182        #####
     183        # Get the headers from the META information.
     184        headers = []
     185        if 'HTTP_CONTENT_TYPE' not in META:
     186            headers.append('Content-Type: %s' % (META.get('CONTENT_TYPE', '')))
     187
     188        if 'HTTP_CONTENT_LENGTH' not in META:
     189            headers.append('Content-Length: %s' % (META.get('CONTENT_LENGTH', '0')))
     190
     191        for key, value in META.items():
     192            if key.startswith('HTTP_'):
     193                headers.append('%s: %s' % (key[5:].replace('_','-').title(), value))
     194
     195        raw_message = '\r\n'.join(headers)
     196        raw_message += '\r\n\r\n' + input_data.read()
     197
     198        msg = email.message_from_string(raw_message)
     199        POST = QueryDict('', mutable=True)
     200        FILES = MultiValueDict()
     201        for submessage in msg.get_payload():
     202            if submessage and isinstance(submessage, email.Message.Message):
     203                name_dict = parse_header(submessage['Content-Disposition'])[1]
     204                field_name = force_unicode(name_dict['name'], encoding, errors='replace')
     205
     206                if 'filename' in name_dict:
     207                    assert not isinstance(submessage.get_payload(), list), "Nested MIME messages are not supported"
     208                    if not name_dict['filename'].strip():
     209                        continue
     210
     211                    filename = force_unicode(name_dict['filename'][name_dict['filename'].rfind("\\")+1:],
     212                                             encoding, errors='replace')
     213                    content_type = 'Content-Type' in submessage and submessage['Content-Type'] or None
     214
     215                    file_obj = InMemoryUploadedFile(StringIO(submessage.get_payload()),
     216                                                    field_name, filename, content_type, None)
     217
     218                    FILES.appendlist(field_name, file_obj)
     219                else:
     220                    content = force_unicode(submessage.get_payload(), encoding, errors='replace')
     221                    POST.appendlist(field_name, content)
     222
     223        return POST, FILES
     224
     225
     226    def new_file(self, field_name, file_name, content_type, content_length, charset):
     227        """
     228        Do Nothing.
     229        """
     230        return
     231
     232    def receive_data_chunk(self, raw_data, start, stop):
     233        """
     234        Do nothing.
     235        """
     236        return raw_data
     237
     238    def file_complete(self, file_size):
     239        """
     240        Do nothing.
     241        """
     242        return
     243
  • django/core/files/filemove.py

     
     1import os
     2
     3__all__ = ('file_move_safe',)
     4
     5try:
     6    import shutil
     7    file_move = shutil.move
     8except ImportError:
     9    file_move = os.rename
     10
     11def file_move_safe(old_file_name, new_file_name, chunk_size = 1024*64, allow_overwrite=False):
     12    """
     13    Moves a file from one location to another in the safest way possible.
     14   
     15    First, it tries using shutils.move, which is OS-dependent but doesn't
     16    break with change of filesystems. Then it tries os.rename, which will
     17    break if it encounters a change in filesystems. Lastly, it streams
     18    it manually from one file to another in python.
     19
     20    Without ``allow_overwrite``, if the destination file exists, the
     21    file will raise an IOError.
     22    """
     23
     24    from django.core.files import filelocks
     25
     26    if old_file_name == new_file_name:
     27        # No file moving takes place.
     28        return
     29
     30    if not allow_overwrite and os.path.exists(new_file_name):
     31        raise IOError, "Django does not allow overwriting files."
     32
     33    try:
     34        file_move(old_file_name, new_file_name)
     35        return
     36    except OSError: # moving to another filesystem
     37        pass
     38
     39    new_file = open(new_file_name, 'wb')
     40    # exclusive lock
     41    filelocks.lock(new_file, filelocks.LOCK_EX)
     42    old_file = open(old_file_name, 'rb')
     43    current_chunk = None
     44
     45    while current_chunk != '':
     46        current_chunk = old_file.read(chunk_size)
     47        new_file.write(current_chunk)
     48
     49    new_file.close()
     50    old_file.close()
     51
     52    os.remove(old_file_name)
  • django/newforms/fields.py

     
    416416
    417417class UploadedFile(StrAndUnicode):
    418418    "A wrapper for files uploaded in a FileField"
    419     def __init__(self, filename, content):
     419    def __init__(self, filename, data):
    420420        self.filename = filename
    421         self.content = content
     421        self.data = data
    422422
    423423    def __unicode__(self):
    424424        """
     
    445445        elif not data and initial:
    446446            return initial
    447447        try:
    448             f = UploadedFile(data['filename'], data['content'])
    449         except TypeError:
     448            f = UploadedFile(data.file_name, data)
     449        except (TypeError, AttributeError):
    450450            raise ValidationError(self.error_messages['invalid'])
    451         except KeyError:
    452             raise ValidationError(self.error_messages['missing'])
    453         if not f.content:
     451        if not f.data.file_size:
    454452            raise ValidationError(self.error_messages['empty'])
    455453        return f
    456454
     
    470468        elif not data and initial:
    471469            return initial
    472470        from PIL import Image
    473         from cStringIO import StringIO
     471
     472        # We need to get the file, it either has a path
     473        # or we have to read it all into memory...
     474        if hasattr(data, 'temporary_file_path'):
     475            file = data.temporary_file_path()
     476        else:
     477            try:
     478                from cStringIO import StringIO
     479            except ImportError:
     480                from StringIO import StringIO
     481            file = StringIO(data.read())
     482
    474483        try:
    475484            # load() is the only method that can spot a truncated JPEG,
    476485            #  but it cannot be called sanely after verify()
    477             trial_image = Image.open(StringIO(f.content))
     486            trial_image = Image.open(file)
    478487            trial_image.load()
    479488            # verify() is the only method that can spot a corrupt PNG,
    480489            #  but it must be called immediately after the constructor
    481             trial_image = Image.open(StringIO(f.content))
     490            trial_image = Image.open(file)
    482491            trial_image.verify()
    483492        except Exception: # Python Imaging Library doesn't recognize it as an image
    484493            raise ValidationError(self.error_messages['invalid_image'])
  • tests/modeltests/model_forms/models.py

     
    7575__test__ = {'API_TESTS': """
    7676>>> from django import newforms as forms
    7777>>> from django.newforms.models import ModelForm
     78>>> from django.core.files.uploadedfile import SimpleUploadedFile
    7879
    7980The bare bones, absolutely nothing custom, basic case.
    8081
     
    792793
    793794# Upload a file and ensure it all works as expected.
    794795
    795 >>> f = TextFileForm(data={'description': u'Assistance'}, files={'file': {'filename': 'test1.txt', 'content': 'hello world'}})
     796>>> f = TextFileForm(data={'description': u'Assistance'}, files={'file': SimpleUploadedFile('test1.txt', 'hello world')})
    796797>>> f.is_valid()
    797798True
    798799>>> type(f.cleaned_data['file'])
     
    819820
    820821# Override the file by uploading a new one.
    821822
    822 >>> f = TextFileForm(data={'description': u'Assistance'}, files={'file': {'filename': 'test2.txt', 'content': 'hello world'}}, instance=instance)
     823>>> f = TextFileForm(data={'description': u'Assistance'}, files={'file': SimpleUploadedFile('test2.txt', 'hello world')}, instance=instance)
    823824>>> f.is_valid()
    824825True
    825826>>> instance = f.save()
     
    838839>>> instance.file
    839840''
    840841
    841 >>> f = TextFileForm(data={'description': u'Assistance'}, files={'file': {'filename': 'test3.txt', 'content': 'hello world'}}, instance=instance)
     842>>> f = TextFileForm(data={'description': u'Assistance'}, files={'file': SimpleUploadedFile('test3.txt', 'hello world')}, instance=instance)
    842843>>> f.is_valid()
    843844True
    844845>>> instance = f.save()
     
    858859
    859860>>> image_data = open(os.path.join(os.path.dirname(__file__), "test.png")).read()
    860861
    861 >>> f = ImageFileForm(data={'description': u'An image'}, files={'image': {'filename': 'test.png', 'content': image_data}})
     862>>> f = ImageFileForm(data={'description': u'An image'}, files={'image': SimpleUploadedFile('test.png', image_data)})
    862863>>> f.is_valid()
    863864True
    864865>>> type(f.cleaned_data['image'])
     
    885886
    886887# Override the file by uploading a new one.
    887888
    888 >>> f = ImageFileForm(data={'description': u'Changed it'}, files={'image': {'filename': 'test2.png', 'content': image_data}}, instance=instance)
     889>>> f = ImageFileForm(data={'description': u'Changed it'}, files={'image': SimpleUploadedFile('test2.png', image_data)}, instance=instance)
    889890>>> f.is_valid()
    890891True
    891892>>> instance = f.save()
     
    904905>>> instance.image
    905906''
    906907
    907 >>> f = ImageFileForm(data={'description': u'And a final one'}, files={'image': {'filename': 'test3.png', 'content': image_data}}, instance=instance)
     908>>> f = ImageFileForm(data={'description': u'And a final one'}, files={'image': SimpleUploadedFile('test3.png', image_data)}, instance=instance)
    908909>>> f.is_valid()
    909910True
    910911>>> instance = f.save()
  • tests/regressiontests/bug639/tests.py

     
    99from regressiontests.bug639.models import Photo
    1010from django.http import QueryDict
    1111from django.utils.datastructures import MultiValueDict
     12from django.core.files.uploadedfile import SimpleUploadedFile
    1213
    1314class Bug639Test(unittest.TestCase):
    1415       
     
    2122       
    2223        # Fake a request query dict with the file
    2324        qd = QueryDict("title=Testing&image=", mutable=True)
    24         qd["image_file"] = {
    25             "filename" : "test.jpg",
    26             "content-type" : "image/jpeg",
    27             "content" : img
    28         }
     25        qd["image_file"] = SimpleUploadedFile('test.jpg', img, 'image/jpeg')
    2926       
    3027        manip = Photo.AddManipulator()
    3128        manip.do_html2python(qd)
     
    3936        Make sure to delete the "uploaded" file to avoid clogging /tmp.
    4037        """
    4138        p = Photo.objects.get()
    42         os.unlink(p.get_image_filename())
    43  No newline at end of file
     39        os.unlink(p.get_image_filename())
  • tests/regressiontests/forms/error_messages.py

     
    11# -*- coding: utf-8 -*-
    22tests = r"""
    33>>> from django.newforms import *
     4>>> from django.core.files.uploadedfile import SimpleUploadedFile
    45
    56# CharField ###################################################################
    67
     
    214215Traceback (most recent call last):
    215216...
    216217ValidationError: [u'INVALID']
    217 >>> f.clean({})
     218>>> f.clean(SimpleUploadedFile('name', None))
    218219Traceback (most recent call last):
    219220...
    220 ValidationError: [u'MISSING']
    221 >>> f.clean({'filename': 'name', 'content':''})
     221ValidationError: [u'EMPTY FILE']
     222>>> f.clean(SimpleUploadedFile('name', ''))
    222223Traceback (most recent call last):
    223224...
    224225ValidationError: [u'EMPTY FILE']
  • tests/regressiontests/forms/fields.py

     
    22tests = r"""
    33>>> from django.newforms import *
    44>>> from django.newforms.widgets import RadioFieldRenderer
     5>>> from django.core.files.uploadedfile import SimpleUploadedFile
    56>>> import datetime
    67>>> import time
    78>>> import re
     
    773774>>> f.clean({})
    774775Traceback (most recent call last):
    775776...
    776 ValidationError: [u'No file was submitted.']
     777ValidationError: [u'No file was submitted. Check the encoding type on the form.']
    777778
    778779>>> f.clean({}, '')
    779780Traceback (most recent call last):
    780781...
    781 ValidationError: [u'No file was submitted.']
     782ValidationError: [u'No file was submitted. Check the encoding type on the form.']
    782783
    783784>>> f.clean({}, 'files/test3.pdf')
    784785'files/test3.pdf'
     
    788789...
    789790ValidationError: [u'No file was submitted. Check the encoding type on the form.']
    790791
    791 >>> f.clean({'filename': 'name', 'content': None})
     792>>> f.clean(SimpleUploadedFile('name', None))
    792793Traceback (most recent call last):
    793794...
    794795ValidationError: [u'The submitted file is empty.']
    795796
    796 >>> f.clean({'filename': 'name', 'content': ''})
     797>>> f.clean(SimpleUploadedFile('name', ''))
    797798Traceback (most recent call last):
    798799...
    799800ValidationError: [u'The submitted file is empty.']
    800801
    801 >>> type(f.clean({'filename': 'name', 'content': 'Some File Content'}))
     802>>> type(f.clean(SimpleUploadedFile('name', 'Some File Content')))
    802803<class 'django.newforms.fields.UploadedFile'>
    803804
    804 >>> type(f.clean({'filename': 'name', 'content': 'Some File Content'}, 'files/test4.pdf'))
     805>>> type(f.clean(SimpleUploadedFile('name', 'Some File Content'), 'files/test4.pdf'))
    805806<class 'django.newforms.fields.UploadedFile'>
    806807
    807808# URLField ##################################################################
  • tests/regressiontests/forms/forms.py

     
    11# -*- coding: utf-8 -*-
    22tests = r"""
    33>>> from django.newforms import *
     4>>> from django.core.files.uploadedfile import SimpleUploadedFile
    45>>> import datetime
    56>>> import time
    67>>> import re
     
    14651466>>> print f
    14661467<tr><th>File1:</th><td><ul class="errorlist"><li>This field is required.</li></ul><input type="file" name="file1" /></td></tr>
    14671468
    1468 >>> f = FileForm(data={}, files={'file1': {'filename': 'name', 'content':''}}, auto_id=False)
     1469>>> f = FileForm(data={}, files={'file1': SimpleUploadedFile('name', '')}, auto_id=False)
    14691470>>> print f
    14701471<tr><th>File1:</th><td><ul class="errorlist"><li>The submitted file is empty.</li></ul><input type="file" name="file1" /></td></tr>
    14711472
     
    14731474>>> print f
    14741475<tr><th>File1:</th><td><ul class="errorlist"><li>No file was submitted. Check the encoding type on the form.</li></ul><input type="file" name="file1" /></td></tr>
    14751476
    1476 >>> f = FileForm(data={}, files={'file1': {'filename': 'name', 'content':'some content'}}, auto_id=False)
     1477>>> f = FileForm(data={}, files={'file1': SimpleUploadedFile('name', 'some content')}, auto_id=False)
    14771478>>> print f
    14781479<tr><th>File1:</th><td><input type="file" name="file1" /></td></tr>
    14791480>>> f.is_valid()
  • tests/regressiontests/test_client_regress/views.py

     
    11from django.contrib.auth.decorators import login_required
    22from django.http import HttpResponse, HttpResponseRedirect, HttpResponseServerError
     3import sha
    34
    45def no_template_view(request):
    56    "A simple view that expects a GET request, and returns a rendered template"
     
    1011    Check that a file upload can be updated into the POST dictionary without
    1112    going pear-shaped.
    1213    """
     14    from django.core.files.uploadedfile import UploadedFile
    1315    form_data = request.POST.copy()
    1416    form_data.update(request.FILES)
    15     if isinstance(form_data['file_field'], dict) and isinstance(form_data['name'], unicode):
     17    if isinstance(form_data['file_field'], UploadedFile) and isinstance(form_data['name'], unicode):
    1618        return HttpResponse('')
    1719    else:
    1820        return HttpResponseServerError()
    1921
     22def file_upload_view_verify(request):
     23    """
     24    Use the sha digest hash to verify the uploaded contents.
     25    """
     26    from django.core.files.uploadedfile import UploadedFile
     27    form_data = request.POST.copy()
     28    form_data.update(request.FILES)
     29    for key, value in form_data.items():
     30        if key.endswith('_hash'):
     31            continue
     32        if key + '_hash' not in form_data:
     33            continue
     34        submitted_hash = form_data[key + '_hash']
     35        if isinstance(value, UploadedFile):
     36            new_hash = sha.new(value.read()).hexdigest()
     37        else:
     38            new_hash = sha.new(value).hexdigest()
     39        if new_hash != submitted_hash:
     40            return HttpResponseServerError()
     41
     42    return HttpResponse('')
     43
    2044def get_view(request):
    2145    "A simple login protected view"
    2246    return HttpResponse("Hello world")
     
    3761def login_protected_redirect_view(request):
    3862    "A view that redirects all requests to the GET view"
    3963    return HttpResponseRedirect('/test_client_regress/get_view/')
    40 login_protected_redirect_view = login_required(login_protected_redirect_view)
    41  No newline at end of file
     64login_protected_redirect_view = login_required(login_protected_redirect_view)
  • tests/regressiontests/test_client_regress/models.py

     
    55from django.test import Client, TestCase
    66from django.core.urlresolvers import reverse
    77import os
     8import sha
    89
    910class AssertContainsTests(TestCase):
    1011    def test_contains(self):
     
    243244        response = self.client.post('/test_client_regress/file_upload/', post_data)
    244245        self.assertEqual(response.status_code, 200)
    245246
     247    def test_large_upload(self):
     248        import tempfile
     249        dir = tempfile.gettempdir()
     250
     251        (fd, name1) = tempfile.mkstemp(suffix='.file1', dir=dir)
     252        file1 = os.fdopen(fd, 'w+b')
     253        file1.write('a' * (2 ** 21))
     254        file1.seek(0)
     255
     256        (fd, name2) = tempfile.mkstemp(suffix='.file2', dir=dir)
     257        file2 = os.fdopen(fd, 'w+b')
     258        file2.write('a' * (10 * 2 ** 20))
     259        file2.seek(0)
     260
     261        post_data = {
     262            'name': 'Ringo',
     263            'file_field1': file1,
     264            'file_field2': file2,
     265            }
     266
     267        for key in post_data.keys():
     268            try:
     269                post_data[key + '_hash'] = sha.new(post_data[key].read()).hexdigest()
     270                post_data[key].seek(0)
     271            except AttributeError:
     272                post_data[key + '_hash'] = sha.new(post_data[key]).hexdigest()
     273
     274        response = self.client.post('/test_client_regress/file_upload_verify/', post_data)
     275
     276        for name in (name1, name2):
     277            try:
     278                os.unlink(name)
     279            except:
     280                pass
     281
     282        self.assertEqual(response.status_code, 200)
     283
     284
    246285class LoginTests(TestCase):
    247286    fixtures = ['testdata']
    248287
  • tests/regressiontests/test_client_regress/urls.py

     
    44urlpatterns = patterns('',
    55    (r'^no_template_view/$', views.no_template_view),
    66    (r'^file_upload/$', views.file_upload_view),
     7    (r'^file_upload_verify/$', views.file_upload_view_verify),
    78    (r'^get_view/$', views.get_view),
    89    url(r'^arg_view/(?P<name>.+)/$', views.view_with_argument, name='arg_view'),
    910    (r'^login_protected_redirect_view/$', views.login_protected_redirect_view)
  • AUTHORS

     
    5858    Jökull Sólberg Auðunsson <jokullsolberg@gmail.com>
    5959    Arthur <avandorp@gmail.com>
    6060    David Avsajanishvili <avsd05@gmail.com>
    61     axiak@mit.edu
     61    Mike Axiak <axiak@mit.edu>
    6262    Niran Babalola <niran@niran.org>
    6363    Morten Bagai <m@bagai.com>
    6464    Mikaël Barbero <mikael.barbero nospam at nospam free.fr>
     
    135135    Marc Fargas <telenieko@telenieko.com>
    136136    Szilveszter Farkas <szilveszter.farkas@gmail.com>
    137137    favo@exoweb.net
     138    fdr <drfarina@gmail.com>
    138139    Dmitri Fedortchenko <zeraien@gmail.com>
    139140    Bill Fenner <fenner@gmail.com>
    140141    Stefane Fermgier <sf@fermigier.com>
  • docs/upload_handling.txt

     
     1===============
     2Upload Handlers
     3===============
     4
     5Upload handlers are components to modify how Django handles file uploads. Through the upload handling framework, Django allows complete control over what happens during and after uploads.
     6
     7An upload handler can do many different things with the uploaded data. For instance, one can stream your uploaded data to disk, while another can generate the data to render a Progress Bar for your uploads.
     8
     9There are two pieces to the Django upload handling: the upload handler and the uploaded file. These are both represented by python classes -- ``FileUploadHandler`` and ``UploadedFile`` respectively. Throughout the lifetime of the upload process, Django will call on the upload handler to handle the upload, while the upload handler is expected to return an ``UploadedFile`` object at the end of a file's upload.
     10
     11Adding Upload Handlers to your Request
     12======================================
     13
     14To use an upload handler, you have to register it in your request. The request object contains a list of handlers (``request.upload_handlers``) that will be called in order. To append a handler to the list, you would append to it like any other list. For example, suppose you had an ``ProgressBarUploadHandler`` class. To append it to your upload handlers you would write::
     15
     16    request.upload_handlers.append(ProgressBarUploadHandler())
     17
     18However, since the progress bar handler would probably need to run before the other handlers get a chance, you'd probably want to insert it. That is, you'd write::
     19
     20   request.upload_handlers.insert(0, ProgressBarUploadHandler())
     21
     22If you want to replace the upload handlers completely, you can just assign a new list::
     23
     24   request.upload_handlers = [ProgressBarUploadHandler()]
     25
     26And all Django will do is keep a progress, but do nothing with the file itself! One more note: After the upload has completed, you are no longer allowed to assign or modify this list, and Django will raise an error if you try to modify this list after an upload.
     27
     28Writing a File Upload Handler
     29=============================
     30
     31All file upload handlers are subclasses of ``FileUploadHandler``, found in ``django.core.files.fileuploadhandler``. To create your handler, you need to define the required methods followed by the methods that make most sense to you:
     32
     33chunk_size
     34----------
     35
     36This is an integer attribute that specifies how large the chunks we should put into memory from the network are. The chunk sizes should be divisible by ``4`` and should not exceed ``2 ** 31 - 1`` in size. When there are multiple chunk sizes provided by multiple handlers, Django will use the smallest chunk size.
     37
     38new_file
     39--------
     40
     41This signals that a new file is starting. You can initialize a file or whatever else is needed on a new file upload.
     42
     43Interface: ``new_file(self, field_name, file_name, content_type, content_length, charset)``
     44
     45``field_name`` is a string name of the field this was POSTed as.
     46
     47``file_name`` is the unicode filename that was provided by the browser.
     48
     49``content_type`` is the MIME type provided by the browser -- E.g. ``'image/jpeg'``.
     50
     51``content_length`` is the length of the image given by the browser if provided, ``None`` otherwise.
     52
     53``charset`` is the charset given by the browser if provided, ``None`` otherwise.
     54
     55Returns: ``None`` if you want other handlers to get ``new_file`` called. Something nonzero if you don't want subsequent handlers to get a chance.
     56
     57receive_data_chunk
     58------------------
     59*required*
     60
     61This method is used to do something with the new chunk of data. For example: The ``TemporaryFileUploadHandler`` takes the data and writes it to disk.
     62
     63Interface: ``receive_data_chunk(self, raw_data, start, stop)``
     64
     65``raw_data`` is a byte string containing the uploaded data.
     66
     67``start`` is the byte number where the chunk starts.
     68
     69``stop`` is the byte number where the chunk stops.
     70
     71Returns: ``None`` if you don't want the subsequent upload handlers to receive the data. Whatever else you return gets fed into the subsequent upload handlers' ``receive_data_chunk`` method.
     72
     73Exceptions: If you raise a ``StopUpload`` or a ``SkipFile`` exception, the upload will abort or the file will be skipped respectively.
     74
     75file_complete
     76-------------
     77*required*
     78
     79This method defines when a function has finished uploading and gets packaged into an ``UploadedFile`` object.
     80
     81Interface: ``file_complete(self, file_size)``
     82
     83``file_size`` is the number of bytes you have received for this file.
     84
     85Returns: An ``UploadedFile`` object to set in the ``request.FILES`` dictionary. ``None`` if you want the subsequent upload handlers to get an ``UploadedFile`` object.
     86
     87upload_complete
     88---------------
     89
     90This method defines when the entire upload has completed.
     91
     92Interface: ``upload_complete(self)``
     93
     94handle_raw_input
     95----------------
     96
     97This method allows the handler to completely override the parsing of the raw HTTP-layered input.
     98
     99Interface: ``handle_raw_input(self, input_data, META, content_length, boundary, encoding)``
     100
     101``input_data`` is a file-like object that supports the read operation.
     102
     103``META`` is the same object as ``request.META``.
     104
     105``content_length`` is the length of the data in ``input_data``. Don't read more than ``content_length`` bytes from ``input_data``.
     106
     107``boundary`` is the MIME boundary for this request.
     108
     109``encoding`` is the encoding of the request.
     110
     111Returns: ``None`` if you want to go on to the next stage, ``(POST, FILES)`` if you want to return the new data structures suitable for the request directly.
     112
     113Defining an Uploaded File
     114=========================
     115
     116All file upload handlers are subclasses of ``UploadedFile``, found in ``django.core.files.uploadedfile``. The uploaded file object is returned by the handler above in ``file_complete``. To create your own uploaded file class, you need to define the required methods followed by the methods that make most sense to you:
     117
     118read
     119----
     120*required*
     121
     122Interface: ``read(self, num_bytes=None)``
     123
     124Returns: A byte string of length ``num_bytes`` (or the size of the file if ``num_bytes`` is not supplied).
     125
     126chunk
     127-----
     128
     129A generator to yield small chunks from the file. With the ``read()`` defined, the ``UploadedFile`` class already defines a ``chunk()`` method that's probably suitable.
     130
     131Interface: ``chunk(self, chunk_size=None)``
     132
     133
     134multiple_chunks
     135---------------
     136
     137Interface: ``multiple_chunks(self, chunk_size=None)``
     138
     139Returns: ``True`` or ``False`` depending on whether or not the user of this file can expect more than one chunk when calling ``chunk(self, chunk_size)``. If all the data is in memory, you should return ``False``.
     140
     141temporary_file_path
     142-------------------
     143
     144If defined, this method should return the file path of the file on the operating system. This will let users move files rather than write to new files if the option is available.
     145
     146Interface: ``temporary_file_path(self)``
     147
     148Returns: A file path in a file system on the local computer.
  • docs/settings.txt

     
    513513The character encoding used to decode any files read from disk. This includes
    514514template files and initial SQL data files.
    515515
     516FILE_UPLOAD_TEMP_DIR
     517--------------------
     518
     519**New in Django development version**
     520
     521Default: ``None``
     522
     523The directory to store data temporarily while uploading files. If ``None``, Django will use the standard temporary directory for the operating system. For example, this will default to '/tmp' on *nix-style operating systems.
     524
    516525FIXTURE_DIRS
    517526-------------
    518527
Back to Top