Ticket #2070: 2070_revision7484.diff

File 2070_revision7484.diff, 99.2 KB (added by axiak, 7 years ago)

Updated diff per Jacob's requests and useful feedback.

  • django/test/client.py

     
    1818BOUNDARY = 'BoUnDaRyStRiNg'
    1919MULTIPART_CONTENT = 'multipart/form-data; boundary=%s' % BOUNDARY
    2020
     21
     22class FakePayload(object):
     23    """
     24    A wrapper around StringIO that restricts what can be read,
     25    since data from the network can't be seeked and cannot
     26    be read outside of its content length (or else we hang).
     27    """
     28    def __init__(self, content):
     29        self.__content = StringIO(content)
     30        self.__len = len(content)
     31
     32    def read(self, num_bytes=None):
     33        if num_bytes is None:
     34            num_bytes = self.__len or 1
     35        assert self.__len >= num_bytes, "Cannot read more than the available bytes from the HTTP incoming data."
     36        content = self.__content.read(num_bytes)
     37        self.__len -= num_bytes
     38        return content
     39
     40
    2141class ClientHandler(BaseHandler):
    2242    """
    2343    A HTTP Handler that can be used for testing purposes.
     
    230250            'CONTENT_TYPE':   content_type,
    231251            'PATH_INFO':      urllib.unquote(path),
    232252            'REQUEST_METHOD': 'POST',
    233             'wsgi.input':     StringIO(post_data),
     253            'wsgi.input':     FakePayload(post_data),
    234254        }
    235255        r.update(extra)
    236256
  • django/http/multipartparser.py

     
     1"""
     2MultiPart parsing for file uploads.
     3
     4This object will take the file upload headers
     5and the file upload handler and chunk the upload
     6data for the handler to deal with.
     7"""
     8from django.utils.datastructures import MultiValueDict
     9from django.utils.encoding import force_unicode
     10from django.utils.text import unescape_entities
     11
     12__all__ = ('MultiPartParser','MultiPartParserError','InputStreamExhausted')
     13
     14class MultiPartParserError(Exception):
     15    pass
     16
     17class InputStreamExhausted(Exception):
     18    """ No more reads are allowed from this device. """
     19    pass
     20
     21
     22class FieldType(object):
     23    """
     24    Denotes what type of field a given section of the multipart
     25    stream is.
     26
     27    The types can be one of ("FILE", "RAW", "FIELD")
     28    """
     29
     30    def __init__(self, name):
     31        self.name = name
     32
     33    def __str__(self):
     34        return self.name
     35
     36RAW = FieldType('RAW')
     37FILE = FieldType('FILE')
     38FIELD = FieldType('FIELD')
     39
     40
     41class MultiPartParser(object):
     42    """
     43    A rfc2388 multipart/form-data parser.
     44
     45    parse() reads the input stream in chunk_size chunks and returns a
     46    tuple of (POST MultiValueDict, FILES MultiValueDict). If
     47    file_upload_dir is defined files will be streamed to temporary
     48    files in the specified directory.
     49    """
     50    def __init__(self, META, input_data, upload_handlers, encoding=None):
     51        """
     52        Initialize the MultiPartParser object.
     53
     54        *META* -- The standard META dictionary in Django request objects.
     55        *input_data* -- The raw post data, as a bytestring.
     56        *upload_handler* -- An object of type UploadHandler
     57                            that performs operations on the uploaded
     58                            data.
     59        *encoding* -- The encoding with which to treat the incoming data.
     60        """
     61        # Import cgi utilities for (near) future use.
     62        global valid_boundary, settings
     63        from django.conf import settings
     64        from cgi import valid_boundary
     65
     66        #
     67        # Content-Type should containt multipart and the boundary information.
     68        #
     69
     70        content_type = META.get('HTTP_CONTENT_TYPE', META.get('CONTENT_TYPE', ''))
     71        if not content_type.startswith('multipart/'):
     72            raise MultiPartParserError('Invalid Content-Type: %s' %
     73                                       content_type)
     74
     75        # Parse the header to get the boundary to split the parts.
     76        ctypes, opts = parse_header(content_type)
     77        boundary = opts.get('boundary')
     78        if not boundary or not valid_boundary(boundary):
     79            raise MultiPartParserError('Invalid boundary in multipart: %s' %
     80                                       boundary)
     81
     82
     83        #
     84        # Content-Length should contain the length of the body we are about
     85        # to receive.
     86        #
     87        try:
     88            content_length = int(META.get('HTTP_CONTENT_LENGTH',
     89                                          META.get('CONTENT_LENGTH',0)))
     90        except (ValueError, TypeError):
     91            # For now set it to 0...we'll try again later on down.
     92            content_length = 0
     93
     94        if content_length <= 0:
     95            # This means we shouldn't continue...raise an error.
     96            raise MultiPartParserError("Invalid content length: %r" % content_length)
     97
     98        self._boundary = boundary
     99        self._input_data = input_data
     100
     101        # For compatibility with low-level network APIs (with 32-bit integers),
     102        # the chunk size should be < 2^31, but still divisible by 4.
     103        self._chunk_size = min(2147483644, *[x.chunk_size for x in upload_handlers
     104                                            if x.chunk_size])
     105
     106        self._meta = META
     107        self._encoding = encoding or settings.DEFAULT_CHARSET
     108        self._content_length = content_length
     109        self._upload_handlers = upload_handlers
     110
     111
     112    def parse(self):
     113        """
     114        Parse the POST data and break it into a FILES MultiValueDict
     115        and a POST MultiValueDict.
     116
     117           *returns* -- A tuple containing the POST and FILES dictionary,
     118                        respectively.
     119        """
     120        from django.core.files.uploadhandler import StopUpload, SkipFile
     121        from django.http import QueryDict
     122
     123        encoding = self._encoding
     124        handlers = self._upload_handlers
     125
     126        limited_input_data = LimitBytes(self._input_data, self._content_length)
     127
     128        # See if the handler will want to take care of the parsing.
     129        # This allows overriding everything if somebody wants it.
     130        for handler in handlers:
     131            result = handler.handle_raw_input(limited_input_data,
     132                                              self._meta,
     133                                              self._content_length,
     134                                              self._boundary,
     135                                              encoding)
     136            if result is not None:
     137                return result[0], result[1]
     138
     139        # Create the data structures to be used later.
     140        self._post = QueryDict('', mutable=True)
     141        self._files = MultiValueDict()
     142
     143        # Instantiate the parser and stream:
     144        stream = LazyStream(ChunkIter(limited_input_data, self._chunk_size))
     145
     146        # Whether or not to signal a file-completion at the beginning of the loop.
     147        old_field_name = None
     148        counters = [0] * len(handlers)
     149
     150        try:
     151            for item_type, meta_data, field_stream in Parser(stream, self._boundary):
     152                if old_field_name:
     153                    # We run this at the beginning of the next loop
     154                    # since we cannot be sure a file is complete until
     155                    # we hit the next boundary/part of the multipart content.
     156                    self.handle_file_complete(old_field_name, counters)
     157
     158                try:
     159                    disposition = meta_data['content-disposition'][1]
     160                    field_name = disposition['name'].strip()
     161                except (KeyError, IndexError, AttributeError):
     162                    continue
     163
     164                transfer_encoding = meta_data.get('content-transfer-encoding')
     165                field_name = force_unicode(field_name, encoding, errors='replace')
     166
     167                if item_type is FIELD:
     168                    # This is a post field, we can just set it in the post
     169                    if transfer_encoding == 'base64':
     170                        raw_data = field_stream.read()
     171                        try:
     172                            data = str(raw_data).decode('base64')
     173                        except:
     174                            data = raw_data
     175                    else:
     176                        data = field_stream.read()
     177
     178                    self._post.appendlist(field_name,
     179                                          force_unicode(data, encoding, errors='replace'))
     180                elif item_type is FILE:
     181                    # This is a file, use the handler...
     182                    file_successful = True
     183                    file_name = disposition.get('filename')
     184                    if not file_name:
     185                        continue
     186                    file_name = force_unicode(file_name, encoding, errors='replace')
     187                    file_name = self.IE_sanitize(unescape_entities(file_name))
     188
     189
     190                    content_type = meta_data.get('content-type', ('',))[0].strip()
     191                    try:
     192                        charset = meta_data.get('content-type', (0,{}))[1].get('charset', None)
     193                    except:
     194                        charset = None
     195
     196                    try:
     197                        content_length = int(meta_data.get('content-length')[0])
     198                    except (IndexError, TypeError, ValueError):
     199                        content_length = None
     200
     201                    counters = [0] * len(handlers)
     202                    try:
     203                        for handler in handlers:
     204                            retval = handler.new_file(field_name, file_name,
     205                                                      content_type, content_length,
     206                                                      charset)
     207                            if retval:
     208                                break
     209
     210                        for chunk in field_stream:
     211                            if transfer_encoding == 'base64':
     212                                # We only special-case base64 transfer encoding
     213                                try:
     214                                    chunk = str(chunk).decode('base64')
     215                                except Exception, e:
     216                                    # Since this is only a chunk, any error is an unfixable error.
     217                                    raise MultiValueParseError("Could not decode base64 data: %r" % e)
     218
     219                            for i, handler in enumerate(handlers):
     220                                chunk_length = len(chunk)
     221                                chunk = handler.receive_data_chunk(chunk,
     222                                                                   counters[i])
     223                                counters[i] += chunk_length
     224                                if chunk is None:
     225                                    # If the chunk received by the handler is None, then don't continue.
     226                                    break
     227
     228                    except SkipFile, e:
     229                        file_successful = False
     230                        # Just use up the rest of this file...
     231                        exhaust(field_stream)
     232                    else:
     233                        # Handle file upload completions on next iteration.
     234                        old_field_name = field_name
     235                else:
     236                    # If this is neither a FIELD or a FILE, just exhaust the stream.
     237                    exhaust(stream)
     238        except StopUpload, e:
     239            if not e.connection_reset:
     240                exhaust(limited_input_data)
     241        else:
     242            # Make sure that the request data is all fed
     243            exhaust(limited_input_data)
     244
     245        # Signal that the upload has completed.
     246        for handler in handlers:
     247            retval = handler.upload_complete()
     248            if retval:
     249                break
     250
     251        return self._post, self._files
     252
     253    def handle_file_complete(self, old_field_name, counters):
     254        """
     255        Handle all the signalling that takes place when a file is complete.
     256        """
     257        for i, handler in enumerate(self._upload_handlers):
     258            file_obj = handler.file_complete(counters[i])
     259            if file_obj:
     260                # If it returns a file object, then set the files dict.
     261                self._files.appendlist(force_unicode(old_field_name,
     262                                                     self._encoding,
     263                                                     errors='replace'),
     264                                       file_obj)
     265                break
     266
     267    def IE_sanitize(self, filename):
     268        """cleanup filename from IE full paths"""
     269        return filename and filename[filename.rfind("\\")+1:].strip()
     270
     271
     272class LazyStream(object):
     273    """
     274    The LazyStream wrapper allows one to pull and "unget" bytes from a stream.
     275
     276    Given a producer object (an iterator that yields bytestrings), the
     277    LazyStream object will support iteration, reading, and keeping a
     278    "look-back" variable in case you need to "unget" some bytes.
     279    """
     280    def __init__(self, producer, length=None):
     281        """
     282        Every LazyStream must have a producer when instantiated.
     283
     284        A producer is an iterable that returns a string each time it
     285        is called.
     286        """
     287        self._producer = producer
     288        self._empty = False
     289        self._leftover = ''
     290        self.length = length
     291        self._position = 0
     292        self._remaining = length
     293
     294        # These fields are to do sanity checking to make sure we don't
     295        # have infinite loops getting/ungetting from the stream. The
     296        # purpose overall is to raise an exception if we perform lots
     297        # of stream get/unget gymnastics without getting
     298        # anywhere. Naturally this is not sound, but most probably
     299        # would indicate a bug if the exception is raised.
     300
     301        # largest position tell us how far this lazystream has ever
     302        # been advanced
     303        self._largest_position = 0
     304
     305        # "modifications since" will start at zero and increment every
     306        # time the position is modified but a new largest position is
     307        # not achieved.
     308        self._modifications_since = 0
     309
     310    def tell(self):
     311        return self.position
     312
     313    def read(self, size=None):
     314        def parts():
     315            remaining = (size is not None and [size] or [self._remaining])[0]
     316            # do the whole thing in one shot if no limit was provided.
     317            if remaining is None:
     318                yield ''.join(self)
     319                return
     320
     321            # otherwise do some bookkeeping to return exactly enough
     322            # of the stream and stashing any extra content we get from
     323            # the producer
     324            while remaining != 0:
     325                assert remaining > 0, 'remaining bytes to read should never go negative'
     326
     327                chunk = self.next()
     328
     329                emitting = chunk[:remaining]
     330                self.unget(chunk[remaining:])
     331                remaining -= len(emitting)
     332                yield emitting
     333
     334        out = ''.join(parts())
     335        return out
     336
     337    def next(self):
     338        """
     339        Used when the exact number of bytes to read is unimportant.
     340
     341        This procedure just returns whatever is chunk is conveniently
     342        returned from the iterator instead. Useful to avoid
     343        unnecessary bookkeeping if performance is an issue.
     344        """
     345        if self._leftover:
     346            output = self._leftover
     347            self._leftover = ''
     348        else:
     349            output = self._producer.next()
     350        self.position += len(output)
     351        return output
     352
     353    def close(self):
     354        """
     355        Used to invalidate/disable this lazy stream.
     356
     357        Replaces the producer with an empty list. Any leftover bytes
     358        that have already been read will still be reported upon read()
     359        and/or next().
     360        """
     361        self._producer = []
     362
     363    def __iter__(self):
     364        return self
     365
     366    def unget(self, bytes):
     367        """
     368        Places bytes back onto the front of the lazy stream.
     369
     370        Future calls to read() will return those bytes first. The
     371        stream position and thus tell() will be rewound.
     372        """
     373        self.position -= len(bytes)
     374        self._leftover = ''.join([bytes, self._leftover])
     375
     376    def _set_position(self, value):
     377        if value > self._largest_position:
     378            self._modifications_since = 0
     379            self._largest_position = value
     380        else:
     381            self._modifications_since += 1
     382            if self._modifications_since > 500:
     383                raise MultiPartParserError("LazyStream thinks it is somehow stuck. Report this to the Django developers\n" + repr(vars(self)))
     384
     385        self._position = value
     386
     387    position = property(lambda self: self._position, _set_position)
     388
     389class ChunkIter(object):
     390    """
     391    An iterable that will yield chunks of data.
     392    Given a file-like object as the constructor,
     393    this object will yield chunks of read operations
     394    from that object.
     395    """
     396    def __init__(self, flo, chunk_size=64 * 1024):
     397        self.flo = flo
     398        self.chunk_size = chunk_size
     399
     400    def next(self):
     401        try:
     402            data = self.flo.read(self.chunk_size)
     403        except InputStreamExhausted:
     404            raise StopIteration()
     405        if data:
     406            return data
     407        else:
     408            raise StopIteration()
     409
     410    def __iter__(self):
     411        return self
     412
     413
     414class LimitBytes(object):
     415    """ Limit bytes for a file object. """
     416    def __init__(self, fileobject, length):
     417        self._file = fileobject
     418        self.remaining = length
     419
     420    def read(self, num_bytes=None):
     421        """
     422        Read data from the underlying file.
     423        If you ask for too much or there isn't anything left,
     424        this will raise an InputStreamExhausted error.
     425        """
     426        if self.remaining <= 0:
     427            raise InputStreamExhausted()
     428        if num_bytes is None:
     429            num_bytes = self.remaining
     430        else:
     431            num_bytes = min(num_bytes, self.remaining)
     432        self.remaining -= num_bytes
     433        return self._file.read(num_bytes)
     434
     435class InterBoundaryIter(object):
     436    """
     437    A Producer that will iterate over boundaries.
     438    """
     439    def __init__(self, stream, boundary):
     440        self._stream = stream
     441        self._boundary = boundary
     442
     443    def __iter__(self):
     444        return self
     445
     446    def next(self):
     447        try:
     448            return LazyStream(BoundaryIter(self._stream, self._boundary))
     449        except InputStreamExhausted:
     450            raise StopIteration()
     451
     452
     453class BoundaryIter(object):
     454    """
     455    A Producer that is sensitive to boundaries.
     456
     457    Will happily yield bytes until a boundary is found. Will yield the
     458    bytes before the boundary, throw away the boundary bytes
     459    themselves, and push the post-boundary bytes back on the stream.
     460
     461    The future calls to .next() after locating the boundary will raise
     462    a StopIteration exception.
     463    """
     464
     465    def __init__(self, stream, boundary):
     466        self._stream = stream
     467        self._boundary = boundary
     468        self._done = False
     469        # rollback an additional six bytes because the format is like
     470        # this: CRLF<boundary>[--CRLF]
     471        self._rollback = len(boundary) + 6
     472
     473        # Try to use mx fast string search if available. Otherwise
     474        # use Python find. Wrap the latter for consistency.
     475        unused_char = self._stream.read(1)
     476        if not unused_char:
     477            raise InputStreamExhausted()
     478        self._stream.unget(unused_char)
     479        try:
     480            from mx.TextTools import FS
     481            self._fs = FS(boundary).find
     482        except ImportError:
     483            self._fs = lambda data: data.find(boundary)
     484
     485    def __iter__(self):
     486        return self
     487
     488    def next(self):
     489        if self._done:
     490            raise StopIteration()
     491
     492        stream = self._stream
     493        rollback = self._rollback
     494
     495        bytes_read = 0
     496        chunks = []
     497        for bytes in stream:
     498            bytes_read += len(bytes)
     499            chunks.append(bytes)
     500            if bytes_read > rollback:
     501                break
     502            if not bytes:
     503                break
     504        else:
     505            self._done = True
     506
     507        if not chunks:
     508            raise StopIteration()
     509
     510        chunk = ''.join(chunks)
     511        boundary = self._find_boundary(chunk, len(chunk) < self._rollback)
     512
     513        if boundary:
     514            end, next = boundary
     515            stream.unget(chunk[next:])
     516            self._done = True
     517            return chunk[:end]
     518        else:
     519            # make sure we dont treat a partial boundary (and
     520            # its separators) as data
     521            if not chunk[:-rollback]:# and len(chunk) >= (len(self._boundary) + 6):
     522                # There's nothing left, we should just return and mark as done.
     523                self._done = True
     524                return chunk
     525            else:
     526                stream.unget(chunk[-rollback:])
     527                return chunk[:-rollback]
     528
     529    def _find_boundary(self, data, eof = False):
     530        """
     531        Finds a multipart boundary in data.
     532
     533        Should no boundry exist in the data None is returned
     534        instead. Otherwise a tuple containing
     535        the indices of the following are returned:
     536
     537         * the end of current encapsulation
     538
     539         * the start of the next encapsulation
     540        """
     541        index = self._fs(data)
     542        if index < 0:
     543            return None
     544        else:
     545            end = index
     546            next = index + len(self._boundary)
     547            data_len = len(data) - 1
     548            # backup over CRLF
     549            if data[max(0,end-1)] == '\n':
     550                end -= 1
     551            if data[max(0,end-1)] == '\r':
     552                end -= 1
     553            # skip over --CRLF
     554            #if data[min(data_len,next)] == '-':
     555            #    next += 1
     556            #if data[min(data_len,next)] == '-':
     557            #    next += 1
     558            #if data[min(data_len,next)] == '\r':
     559            #    next += 1
     560            #if data[min(data_len,next)] == '\n':
     561            #    next += 1
     562            return end, next
     563
     564def exhaust(stream_or_iterable):
     565    """
     566    Completely exhausts an iterator or stream.
     567
     568    Raise a MultiPartParserError if the argument is not a stream or an
     569    iterable.
     570    """
     571    iterator = None
     572    try:
     573        iterator = iter(stream_or_iterable)
     574    except TypeError:
     575        iterator = ChunkIter(stream_or_iterable, 16384)
     576
     577    if iterator is None:
     578        raise MultiPartParserError('multipartparser.exhaust() was passed a non-iterable or stream parameter')
     579
     580    for __ in iterator:
     581        pass
     582
     583def ParseBoundaryStream(stream, max_header_size):
     584        """
     585        Parses one and exactly one stream that encapsulates a boundary.
     586        """
     587        # Stream at beginning of header, look for end of header
     588        # and parse it if found. The header must fit within one
     589        # chunk.
     590        chunk = stream.read(max_header_size)
     591        # 'find' returns the top of these four bytes, so we'll
     592        # need to munch them later to prevent them from polluting
     593        # the payload.
     594        header_end = chunk.find('\r\n\r\n')
     595
     596        def _parse_header(line):
     597            main_value_pair, params = parse_header(line)
     598            try:
     599                name, value = main_value_pair.split(':', 1)
     600            except:
     601                raise ValueError("Invalid header: %r" % line)
     602            return name, (value, params)
     603
     604        if header_end == -1:
     605            # we find no header, so we just mark this fact and pass on
     606            # the stream verbatim
     607            stream.unget(chunk)
     608            return (RAW, {}, stream)
     609
     610        header = chunk[:header_end]
     611
     612        # here we place any excess chunk back onto the stream, as
     613        # well as throwing away the CRLFCRLF bytes from above.
     614        stream.unget(chunk[header_end + 4:])
     615
     616        TYPE = RAW
     617        outdict = {}
     618
     619        # Eliminate blank lines
     620        for line in header.split('\r\n'):
     621            # This terminology ("main value" and "dictionary of
     622            # parameters") is from the Python docs.
     623            try:
     624                name, (value, params) = _parse_header(line)
     625            except:
     626                continue
     627
     628            if name == 'content-disposition':
     629                TYPE = FIELD
     630                if params.get('filename'):
     631                    TYPE = FILE
     632
     633            outdict[name] = value, params
     634
     635        if TYPE == RAW:
     636            stream.unget(chunk)
     637
     638        return (TYPE, outdict, stream)
     639
     640
     641class Parser(object):
     642    def __init__(self, stream, boundary):
     643        self._stream = stream
     644        self._separator = '--' + boundary
     645
     646    def __iter__(self):
     647
     648        boundarystream = InterBoundaryIter(self._stream,
     649                                           self._separator)
     650
     651        for sub_stream in boundarystream:
     652            # Iterate over each part
     653            yield ParseBoundaryStream(sub_stream, 1024)
     654
     655def parse_header(line):
     656    """ Parse the header into a key-value. """
     657    plist = _parse_header_params(';' + line)
     658    key = plist.pop(0).lower()
     659    pdict = {}
     660    for p in plist:
     661        i = p.find('=')
     662        if i >= 0:
     663            name = p[:i].strip().lower()
     664            value = p[i+1:].strip()
     665            if len(value) >= 2 and value[0] == value[-1] == '"':
     666                value = value[1:-1]
     667                value = value.replace('\\\\', '\\').replace('\\"', '"')
     668            pdict[name] = value
     669    return key, pdict
     670
     671def _parse_header_params(s):
     672    plist = []
     673    while s[:1] == ';':
     674        s = s[1:]
     675        end = s.find(';')
     676        while end > 0 and s.count('"', 0, end) % 2:
     677            end = s.find(';', end + 1)
     678        if end < 0:
     679            end = len(s)
     680        f = s[:end]
     681        plist.append(f.strip())
     682        s = s[end:]
     683    return plist
  • django/http/__init__.py

     
    99except ImportError:
    1010    from cgi import parse_qsl
    1111
    12 from django.utils.datastructures import MultiValueDict, FileDict
     12from django.utils.datastructures import MultiValueDict, ImmutableList
    1313from django.utils.encoding import smart_str, iri_to_uri, force_unicode
    14 
     14from django.http.multipartparser import MultiPartParser
    1515from utils import *
    1616
    1717RESERVED_CHARS="!*'();:@&=+$,/?%#[]"
     
    2525
    2626    # The encoding used in GET/POST dicts. None means use default setting.
    2727    _encoding = None
     28    _upload_handlers = ()
    2829
    2930    def __init__(self):
    3031        self.GET, self.POST, self.COOKIES, self.META, self.FILES = {}, {}, {}, {}, {}
     
    102103
    103104    encoding = property(_get_encoding, _set_encoding)
    104105
    105 def parse_file_upload(header_dict, post_data):
    106     """Returns a tuple of (POST QueryDict, FILES MultiValueDict)."""
    107     import email, email.Message
    108     from cgi import parse_header
    109     raw_message = '\r\n'.join(['%s:%s' % pair for pair in header_dict.items()])
    110     raw_message += '\r\n\r\n' + post_data
    111     msg = email.message_from_string(raw_message)
    112     POST = QueryDict('', mutable=True)
    113     FILES = MultiValueDict()
    114     for submessage in msg.get_payload():
    115         if submessage and isinstance(submessage, email.Message.Message):
    116             name_dict = parse_header(submessage['Content-Disposition'])[1]
    117             # name_dict is something like {'name': 'file', 'filename': 'test.txt'} for file uploads
    118             # or {'name': 'blah'} for POST fields
    119             # We assume all uploaded files have a 'filename' set.
    120             if 'filename' in name_dict:
    121                 assert type([]) != type(submessage.get_payload()), "Nested MIME messages are not supported"
    122                 if not name_dict['filename'].strip():
    123                     continue
    124                 # IE submits the full path, so trim everything but the basename.
    125                 # (We can't use os.path.basename because that uses the server's
    126                 # directory separator, which may not be the same as the
    127                 # client's one.)
    128                 filename = name_dict['filename'][name_dict['filename'].rfind("\\")+1:]
    129                 FILES.appendlist(name_dict['name'], FileDict({
    130                     'filename': filename,
    131                     'content-type': 'Content-Type' in submessage and submessage['Content-Type'] or None,
    132                     'content': submessage.get_payload(),
    133                 }))
    134             else:
    135                 POST.appendlist(name_dict['name'], submessage.get_payload())
    136     return POST, FILES
     106    def _initialize_handlers(self):
     107        from django.conf import settings
     108        from django.core.files.uploadhandler import load_handler
     109        handlers = []
     110        # We go through each handler in the settings variable
     111        # and instantiate the handler by calling HandlerClass(request).
     112        for handler in settings.FILE_UPLOAD_HANDLERS:
     113            handlers.append(load_handler(handler, self))
     114        self._upload_handlers = handlers
    137115
     116    def _set_upload_handlers(self, upload_handlers):
     117        """
     118        Set the upload handler to the new handler given in the parameter.
     119        """
     120        if hasattr(self, '_files'):
     121            raise AttributeError("You cannot set the upload handlers after the upload has been processed.")
     122        self._upload_handlers = upload_handlers
    138123
     124    def _get_upload_handlers(self):
     125        if not self._upload_handlers:
     126            # If thre are no upload handlers defined, initialize them from settings.
     127            self._initialize_handlers()
     128        return self._upload_handlers
     129
     130    upload_handlers = property(_get_upload_handlers, _set_upload_handlers)
     131
     132    def parse_file_upload(self, META, post_data):
     133        """Returns a tuple of (POST QueryDict, FILES MultiValueDict)."""
     134        self.upload_handlers = ImmutableList(self.upload_handlers,
     135                                             warning="You cannot alter the upload handlers after the upload has been processed.")
     136        parser = MultiPartParser(META, post_data, self.upload_handlers,
     137                                 self.encoding)
     138        return parser.parse()
     139
     140
    139141class QueryDict(MultiValueDict):
    140142    """
    141143    A specialized MultiValueDict that takes a query string when initialized.
  • django/oldforms/__init__.py

     
    680680        self.field_name, self.is_required = field_name, is_required
    681681        self.validator_list = [self.isNonEmptyFile] + validator_list
    682682
    683     def isNonEmptyFile(self, field_data, all_data):
     683    def isNonEmptyFile(self, new_data, all_data):
     684        if hasattr(new_data, 'upload_errors'):
     685            upload_errors = new_data.upload_errors()
     686            if upload_errors:
     687                raise validators.CriticalValidationError, upload_errors
    684688        try:
    685             content = field_data['content']
    686         except TypeError:
    687             raise validators.CriticalValidationError, ugettext("No file was submitted. Check the encoding type on the form.")
    688         if not content:
     689            file_size = new_data.file_size
     690        except AttributeError:
     691            file_size = len(new_data['content'])
     692        if not file_size:
    689693            raise validators.CriticalValidationError, ugettext("The submitted file is empty.")
    690694
    691695    def render(self, data):
    692696        return mark_safe(u'<input type="file" id="%s" class="v%s" name="%s" />' % \
    693697            (self.get_id(), self.__class__.__name__, self.field_name))
    694698
     699    def prepare(self, new_data):
     700        if hasattr(new_data, 'upload_errors'):
     701            upload_errors = new_data.upload_errors()
     702            new_data[self.field_name] = { '_file_upload_error': upload_errors }
     703
    695704    def html2python(data):
    696705        if data is None:
    697706            raise EmptyValue
  • django/db/models/base.py

     
    1919from django.utils.datastructures import SortedDict
    2020from django.utils.functional import curry
    2121from django.utils.encoding import smart_str, force_unicode, smart_unicode
     22from django.core.files.move import file_move_safe
    2223from django.conf import settings
    2324
    2425try:
     
    471472    def _get_FIELD_size(self, field):
    472473        return os.path.getsize(self._get_FIELD_filename(field))
    473474
    474     def _save_FIELD_file(self, field, filename, raw_contents, save=True):
     475    def _save_FIELD_file(self, field, filename, raw_field, save=True):
    475476        directory = field.get_directory_name()
    476477        try: # Create the date-based directory if it doesn't exist.
    477478            os.makedirs(os.path.join(settings.MEDIA_ROOT, directory))
    478479        except OSError: # Directory probably already exists.
    479480            pass
     481
     482        # Put the deprecation warning first since there are multiple
     483        # locations where we use the new and old interface.
     484        if isinstance(raw_field, dict):
     485            import warnings
     486            from django.core.files.uploadedfile import SimpleUploadedFile
     487            raw_field = SimpleUploadedFile.from_dict(raw_field)
     488            warnings.warn("The dictionary usage for files is deprecated. Use the new object interface instead.", DeprecationWarning)
     489        elif isinstance(raw_field, basestring):
     490            import warnings
     491            from django.core.files.uploadedfile import SimpleUploadedFile
     492            raw_field = SimpleUploadedFile(filename, raw_field)
     493            warnings.warn("The string interface for save_FIELD_file is deprecated.", DeprecationWarning)
     494
     495        if filename is None:
     496            filename = raw_field.file_name
     497
    480498        filename = field.get_filename(filename)
    481499
    482500        # If the filename already exists, keep adding an underscore to the name of
     
    493511        setattr(self, field.attname, filename)
    494512
    495513        full_filename = self._get_FIELD_filename(field)
    496         fp = open(full_filename, 'wb')
    497         fp.write(raw_contents)
    498         fp.close()
     514        if hasattr(raw_field, 'temporary_file_path'):
     515            # This file has a file path that we can move.
     516            raw_field.close()
     517            file_move_safe(raw_field.temporary_file_path(), full_filename)
     518        else:
     519            from django.core.files import locks
     520            fp = open(full_filename, 'wb')
     521            # exclusive lock
     522            locks.lock(fp, locks.LOCK_EX)
     523            # This is a normal uploadedfile that we can stream.
     524            for chunk in raw_field.chunk(65535):
     525                fp.write(chunk)
     526            locks.unlock(fp)
     527            fp.close()
    499528
     529
    500530        # Save the width and/or height, if applicable.
    501531        if isinstance(field, ImageField) and (field.width_field or field.height_field):
    502532            from django.utils.images import get_image_dimensions
  • django/db/models/fields/__init__.py

     
    802802        setattr(cls, 'get_%s_filename' % self.name, curry(cls._get_FIELD_filename, field=self))
    803803        setattr(cls, 'get_%s_url' % self.name, curry(cls._get_FIELD_url, field=self))
    804804        setattr(cls, 'get_%s_size' % self.name, curry(cls._get_FIELD_size, field=self))
    805         setattr(cls, 'save_%s_file' % self.name, lambda instance, filename, raw_contents, save=True: instance._save_FIELD_file(self, filename, raw_contents, save))
     805        setattr(cls, 'save_%s_file' % self.name, lambda instance, filename, raw_field, save=True: instance._save_FIELD_file(self, filename, raw_field, save))
    806806        dispatcher.connect(self.delete_file, signal=signals.post_delete, sender=cls)
    807807
    808808    def delete_file(self, instance):
     
    825825        if new_data.get(upload_field_name, False):
    826826            func = getattr(new_object, 'save_%s_file' % self.name)
    827827            if rel:
    828                 func(new_data[upload_field_name][0]["filename"], new_data[upload_field_name][0]["content"], save)
     828                file = new_data[upload_field_name][0]
    829829            else:
    830                 func(new_data[upload_field_name]["filename"], new_data[upload_field_name]["content"], save)
     830                file = new_data[upload_field_name]
    831831
     832            try:
     833                file_name = file.file_name
     834            except AttributeError:
     835                file_name = file['filename']
     836            func(file_name, file, save)
     837
    832838    def get_directory_name(self):
    833839        return os.path.normpath(force_unicode(datetime.datetime.now().strftime(smart_str(self.upload_to))))
    834840
     
    840846    def save_form_data(self, instance, data):
    841847        from django.newforms.fields import UploadedFile
    842848        if data and isinstance(data, UploadedFile):
    843             getattr(instance, "save_%s_file" % self.name)(data.filename, data.content, save=False)
     849            getattr(instance, "save_%s_file" % self.name)(data.filename, data.data, save=False)
    844850
    845851    def formfield(self, **kwargs):
    846852        defaults = {'form_class': forms.FileField}
  • django/conf/global_settings.py

     
    224224# Example: "http://media.lawrence.com"
    225225MEDIA_URL = ''
    226226
     227# A tuple that enumerates the upload handlers
     228# in order.
     229FILE_UPLOAD_HANDLERS = (
     230    'django.core.files.uploadhandler.MemoryFileUploadHandler',
     231    'django.core.files.uploadhandler.TemporaryFileUploadHandler',
     232)
     233
     234# Number of bytes the length of the request can be before it is
     235# streamed to the file system instead of parsed entirely in memory.
     236FILE_UPLOAD_MAX_MEMORY_SIZE = 2621440
     237
     238# Directory to upload streamed files temporarily.
     239# A value of `None` means that it will use the default temporary
     240# directory for the server's operating system.
     241FILE_UPLOAD_TEMP_DIR = None
     242
    227243# Default formatting for date objects. See all available format strings here:
    228244# http://www.djangoproject.com/documentation/templates/#now
    229245DATE_FORMAT = 'N j, Y'
  • django/core/handlers/wsgi.py

     
    112112        # Populates self._post and self._files
    113113        if self.method == 'POST':
    114114            if self.environ.get('CONTENT_TYPE', '').startswith('multipart'):
    115                 header_dict = dict([(k, v) for k, v in self.environ.items() if k.startswith('HTTP_')])
    116                 header_dict['Content-Type'] = self.environ.get('CONTENT_TYPE', '')
    117                 self._post, self._files = http.parse_file_upload(header_dict, self.raw_post_data)
     115                self._raw_post_data = ''
     116                self._post, self._files = self.parse_file_upload(self.META, self.environ['wsgi.input'])
    118117            else:
    119118                self._post, self._files = http.QueryDict(self.raw_post_data, encoding=self._encoding), datastructures.MultiValueDict()
    120119        else:
  • django/core/handlers/modpython.py

     
    5353    def _load_post_and_files(self):
    5454        "Populates self._post and self._files"
    5555        if 'content-type' in self._req.headers_in and self._req.headers_in['content-type'].startswith('multipart'):
    56             self._post, self._files = http.parse_file_upload(self._req.headers_in, self.raw_post_data)
     56            self._raw_post_data = ''
     57            self._post, self._files = self.parse_file_upload(self.META, self._req)
    5758        else:
    5859            self._post, self._files = http.QueryDict(self.raw_post_data, encoding=self._encoding), datastructures.MultiValueDict()
    5960
  • django/core/files/locks.py

     
     1"""
     2Locking portability based partially on example by
     3Jonathan Feignberg <jdf@pobox.com> in python cookbook:
     4    http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/65203
     5Licensed under the Python Software License.
     6
     7Example Usage::
     8
     9    from django.core.files import locks
     10
     11    f = open('./file', 'wb')
     12
     13    locks.lock(f, locks.LOCK_EX)
     14    f.write('Django')
     15    f.close()
     16"""
     17
     18__all__ = ('LOCK_EX','LOCK_SH','LOCK_NB','lock','unlock')
     19
     20system_type = None
     21
     22try:
     23    import win32con
     24    import win32file
     25    import pywintypes
     26    LOCK_EX = win32con.LOCKFILE_EXCLUSIVE_LOCK
     27    LOCK_SH = 0
     28    LOCK_NB = win32con.LOCKFILE_FAIL_IMMEDIATELY
     29    __overlapped = pywintypes.OVERLAPPED()
     30    system_type = 'nt'
     31except (ImportError, AttributeError):
     32    pass
     33
     34try:
     35    import fcntl
     36    LOCK_EX = fcntl.LOCK_EX
     37    LOCK_SH = fcntl.LOCK_SH
     38    LOCK_NB = fcntl.LOCK_NB
     39    system_type = 'posix'
     40except (ImportError, AttributeError):
     41    pass
     42
     43if system_type == 'nt':
     44    def lock(file, flags):
     45        hfile = win32file._get_osfhandle(file.fileno())
     46        win32file.LockFileEx(hfile, flags, 0, -0x10000, __overlapped)
     47
     48    def unlock(file):
     49        hfile = win32file._get_osfhandle(file.fileno())
     50        win32file.UnlockFileEx(hfile, 0, -0x10000, __overlapped)
     51elif system_type == 'posix':
     52    def lock(file, flags):
     53        fcntl.flock(file.fileno(), flags)
     54
     55    def unlock(file):
     56        fcntl.flock(file.fileno(), fcntl.LOCK_UN)
     57else:
     58    # File locking is not supported.
     59    LOCK_EX = LOCK_SH = LOCK_NB = None
     60       
     61    # Dummy functions that don't do anything.
     62    def lock(file, flags):
     63        pass
     64
     65    def unlock(file):
     66        pass
  • django/core/files/uploadedfile.py

     
     1"""
     2The uploaded file objects for Django.
     3This contains the base UploadedFile and the TemporaryUploadedFile
     4derived class.
     5"""
     6
     7__all__ = ('UploadedFile', 'TemporaryUploadedFile', 'InMemoryUploadedFile')
     8
     9class UploadedFile(object):
     10    """
     11    The UploadedFile object behaves somewhat like a file
     12    object and represents some data that the user submitted
     13    and is stored in some form.
     14    """
     15    DEFAULT_CHUNK_SIZE = 64 * 2**10
     16
     17    def __init__(self):
     18        self.file_size = None
     19        self.file_name = None
     20        self.content_type = None
     21        self.charset = None
     22        pass
     23
     24    def chunk(self, chunk_size=None):
     25        """
     26        Read the file to generate chunks of chunk_size bytes.
     27        """
     28        if not chunk_size:
     29            chunk_size = UploadedFile.DEFAULT_CHUNK_SIZE
     30
     31        if hasattr(self, 'seek'):
     32            self.seek(0)
     33        # Assume the pointer is at zero...
     34        counter = self.file_size
     35
     36        while counter > 0:
     37            yield self.read(chunk_size)
     38            counter -= chunk_size
     39
     40
     41    def multiple_chunks(self, chunk_size=None):
     42        """
     43        Return True if you can expect multiple chunks, False otherwise.
     44        Note: If a particular file representation is in memory, then
     45              override this to return False.
     46        """
     47        if not chunk_size:
     48            chunk_size = UploadedFile.DEFAULT_CHUNK_SIZE
     49        return self.file_size < chunk_size
     50       
     51
     52    def read(self, num_bytes=None):
     53        """
     54        Read from the file in whatever representation it has.
     55        """
     56        raise NotImplementedError()
     57
     58    def open(self):
     59        """
     60        Open the file, if one needs to.
     61        """
     62        pass
     63
     64
     65    def close(self):
     66        """
     67        Close the file, if one needs to.
     68        """
     69        pass
     70
     71    def __getitem__(self, key):
     72        """
     73        This maintains backwards compatibility.
     74        """
     75        import warnings
     76        warnings.warn("The dictionary access of uploaded file objects is deprecated. Use the new object interface instead.", DeprecationWarning)
     77        # Dictionary to translate labels
     78        # for backwards compatbility.
     79        # Should be removed at some point.
     80        backwards_translate = {
     81            'filename': 'file_name',
     82            'content-type': 'content_type',
     83            }
     84
     85        if key == 'content':
     86            return self.read()
     87        else:
     88            return getattr(self, backwards_translate.get(key, key))
     89
     90    def __repr__(self):
     91        """
     92        This representation could be anything and can be overridden.
     93        This is mostly done to make it look somewhat useful.
     94        """
     95        _dict = {
     96            'file_name': self.file_name,
     97            'content_type': self.content_type,
     98            'content': '<omitted>',
     99            }
     100        return repr(_dict)
     101
     102
     103class TemporaryUploadedFile(UploadedFile):
     104    """
     105    Upload a file to a temporary file.
     106    """
     107
     108    def __init__(self, file, file_name, content_type, file_size, charset):
     109        self.file = file
     110        self.file_name = file_name
     111        self.path = file.name
     112        self.content_type = content_type
     113        self.file_size = file_size
     114        self.charset = charset
     115        self.file.seek(0)
     116
     117    def temporary_file_path(self):
     118        """
     119        Return the full path of this file.
     120        """
     121        return self.path
     122
     123    def read(self, *args, **kwargs):
     124        return self.file.read(*args, **kwargs)
     125
     126    def open(self):
     127        """
     128        Assume the person meant to seek.
     129        """
     130        self.seek(0)
     131
     132    def seek(self, *args, **kwargs):
     133        self.file.seek(*args, **kwargs)
     134
     135
     136class InMemoryUploadedFile(UploadedFile):
     137    """
     138    Upload a file into memory.
     139    """
     140    def __init__(self, file, field_name, file_name, content_type, charset):
     141        self.file = file
     142        self.field_name = field_name
     143        self.file_name = file_name
     144        self.content_type = content_type
     145        self.charset = charset
     146        self.file.seek(0)
     147
     148    def seek(self, *args, **kwargs):
     149        self.file.seek(*args, **kwargs)
     150
     151    def open(self):
     152        self.seek(0)
     153
     154    def read(self, *args, **kwargs):
     155        return self.file.read(*args, **kwargs)
     156
     157    def chunk(self, chunk_size=None):
     158        """
     159        Return the entirety of the data regardless.
     160        """
     161        self.file.seek(0)
     162        return self.read()
     163
     164    def multiple_chunks(self, chunk_size=None):
     165        """
     166        Since it's in memory, we'll never have multiple chunks.
     167        """
     168        return False
     169
     170
     171class SimpleUploadedFile(InMemoryUploadedFile):
     172    """
     173    A simple representation of a file, which
     174    just has content, size, and a name.
     175    """
     176    def __init__(self, name, content, content_type='text/plain'):
     177        try:
     178            from cStringIO import StringIO
     179        except ImportError:
     180            from StringIO import StringIO
     181        self.file = StringIO(content or '')
     182        self.file_name = name
     183        self.field_name = None
     184        self.file_size = len(content or '')
     185        self.content_type = content_type
     186        self.charset = None
     187        self.file.seek(0)
     188
     189    def from_dict(cls, file_dict):
     190        """
     191        Creates a SimpleUploadedFile object from
     192        a dictionary object with the following keys:
     193           - filename
     194           - content-type
     195           - content
     196        """
     197        return cls(file_dict['filename'],
     198                   file_dict['content'],
     199                   file_dict.get('content-type', 'text/plain'))
     200
     201    from_dict = classmethod(from_dict)
     202
  • django/core/files/__init__.py

     
     1
  • django/core/files/uploadhandler.py

     
     1""" A fileuploadhandler base and default subclass for handling file uploads.
     2"""
     3import os
     4try:
     5    from cStringIO import StringIO
     6except ImportError:
     7    from StringIO import StringIO
     8
     9from django.utils.encoding import force_unicode
     10from django.utils.datastructures import MultiValueDict
     11from django.core.exceptions import ImproperlyConfigured
     12
     13from django.core.files.uploadedfile import TemporaryUploadedFile, InMemoryUploadedFile
     14
     15__all__ = ('UploadFileException','StopUpload', 'SkipFile',
     16           'FileUploadHandler', 'TemporaryFileUploadHandler',
     17           'MemoryFileUploadHandler', 'load_handler')
     18
     19
     20class UploadFileException(Exception):
     21    """ Any error having to do with Uploading Files. """
     22    pass
     23
     24class StopUpload(UploadFileException):
     25    """ This exception is raised when an upload must abort. """
     26    def __init__(self, connection_reset=False):
     27        """
     28        Initializes a StopUpload Exception object.
     29
     30        By specifying connection_reset=True, Django knows
     31        it is okay to halt the upload without feeding
     32        in the rest of the upload to nowhere.
     33        (And thus keeping the browser from the
     34        Connection Reset error.)
     35        """
     36        self.connection_reset = connection_reset
     37
     38    def __unicode__(self):
     39        if self.connection_reset:
     40            return u'StopUpload: Halt current upload.'
     41        else:
     42            return u'StopUpload: Feed in request data.'
     43
     44class SkipFile(UploadFileException):
     45    """ This exception is raised when a file needs to be skipped. """
     46    pass
     47
     48class FileUploadHandler(object):
     49    """ FileUploadHandler will take data and handle file uploads
     50    in a streamed fashion.
     51    """
     52    chunk_size = 64 * 2 ** 10 #: The default chunk size is 64 KB.
     53
     54    def __init__(self, request=None):
     55        " Initialize some local variables. "
     56        self.file_name = None
     57        self.content_type = None
     58        self.content_length = None
     59        self.charset = None
     60        self.request = request
     61
     62    def handle_raw_input(self, input_data, META, content_length, boundary, encoding=None):
     63        """
     64        Handle the raw input from the client.
     65        Parameters:
     66          *input_data* -- An object that supports reading via .read().
     67          *content_length* -- The (integer) value of the Content-Length header from the client.
     68          *boundary* -- The boundary from the Content-Type header. Be sure to prepend two '--'.
     69        """
     70        pass
     71
     72    def new_file(self, field_name, file_name, content_type, content_length, charset=None):
     73        """
     74        Signal that a new file has been started.
     75       
     76        Warning: Do not trust content_length, if you get it at all.
     77        """
     78        self.field_name = field_name
     79        self.file_name = file_name
     80        self.content_type = content_type
     81        self.content_length = content_length
     82        self.charset = charset
     83
     84    def receive_data_chunk(self, raw_data, start):
     85        """
     86        Receive data from the streamed upload parser.
     87        Start is the position in the file of the chunk.
     88        """
     89        raise NotImplementedError()
     90
     91    def file_complete(self, file_size):
     92        """
     93        Signal that a file has completed.
     94        File size corresponds to the actual size accumulated
     95        by all the chunks.
     96
     97        This should return a valid UploadedFile object.
     98        """
     99        raise NotImplementedError()
     100
     101    def upload_complete(self):
     102        """
     103        Signal that the upload is complete.
     104        Do any cleanup that is necessary for this handler.
     105        """
     106        pass
     107
     108
     109
     110class TemporaryFileUploadHandler(FileUploadHandler):
     111    """
     112    Upload the streaming data into a temporary file.
     113    """
     114    def __init__(self, *args, **kwargs):
     115        """ Import settings for later. """
     116        super(TemporaryFileUploadHandler, self).__init__(*args, **kwargs)
     117        global settings
     118        from django.conf import settings
     119
     120    def new_file(self, file_name, *args, **kwargs):
     121        """
     122        Create the file object to append to as data is coming in.
     123        """
     124        super(TemporaryFileUploadHandler, self).new_file(file_name, *args, **kwargs)
     125        self.file = TemporaryFile(settings.FILE_UPLOAD_TEMP_DIR)
     126        self.write = self.file.write
     127
     128    def receive_data_chunk(self, raw_data, start):
     129        """
     130        Once we get the data, we will save it to our file.
     131        """
     132        self.write(raw_data)
     133
     134    def file_complete(self, file_size):
     135        """
     136        Signal that a file has completed.
     137        File size corresponds to the actual size accumulated
     138        by all the chunks.
     139
     140        This should return a valid UploadedFile object.
     141        """
     142        self.file.seek(0)
     143        return TemporaryUploadedFile(self.file, self.file_name,
     144                                     self.content_type, file_size,
     145                                     self.charset)
     146
     147
     148class MemoryFileUploadHandler(FileUploadHandler):
     149    """
     150    The MemoryFileUploadHandler will place the data directly into memory.
     151    """
     152
     153    def handle_raw_input(self, input_data, META, content_length, boundary, encoding=None):
     154        """
     155        Use the content_length to signal whether or not this handler should be in use.
     156        """
     157        global settings
     158        from django.conf import settings
     159
     160        # If the the post is too large, we cannot use the Memory handler.
     161        if content_length > settings.FILE_UPLOAD_MAX_MEMORY_SIZE:
     162            self.activated = False
     163        else:
     164            self.activated = True
     165
     166    def new_file(self, *args, **kwargs):
     167        super(MemoryFileUploadHandler, self).new_file(*args, **kwargs)
     168        if self.activated:
     169            self.file = StringIO()
     170            return "Stop"
     171
     172    def receive_data_chunk(self, raw_data, start):
     173        """
     174        Add the data to the StringIO file.
     175        """
     176        if self.activated:
     177            self.file.write(raw_data)
     178        else:
     179            return raw_data
     180
     181    def file_complete(self, file_size):
     182        """
     183        Return a file object if we're activated.
     184        """
     185        if not self.activated:
     186            return
     187
     188        return InMemoryUploadedFile(self.file, self.field_name, self.file_name,
     189                                    self.content_type, self.charset)
     190
     191
     192class TemporaryFile(object):
     193    """
     194    A temporary file that tries to delete itself when garbage collected.
     195    """
     196    def __init__(self, dir):
     197        import tempfile
     198        if not dir:
     199            dir = tempfile.gettempdir()
     200        try:
     201            (fd, name) = tempfile.mkstemp(suffix='.upload', dir=dir)
     202            self.file = os.fdopen(fd, 'w+b')
     203        except (OSError, IOError):
     204            raise OSError("Could not create temporary file for uploading, have you set settings.FILE_UPLOAD_TEMP_DIR correctly?")
     205        self.name = name
     206
     207    def __getattr__(self, name):
     208        a = getattr(self.__dict__['file'], name)
     209        if type(a) != type(0):
     210            setattr(self, name, a)
     211        return a
     212
     213    def __del__(self):
     214        try:
     215            os.unlink(self.name)
     216        except OSError:
     217            pass
     218
     219def load_handler(path, *args, **kwargs):
     220    """
     221    Given a path to a handler return the instantiation of that handler.
     222    E.g.::
     223        load_handler('django.core.files.uploadhandler.TemporaryFileUploadHandler', request)
     224    will return a TemporaryFileUploadHandler object.
     225    """
     226    i = path.rfind('.')
     227    module, attr = path[:i], path[i+1:]
     228    try:
     229        mod = __import__(module, {}, {}, [attr])
     230    except ImportError, e:
     231        raise ImproperlyConfigured('Error importing upload handler module %s: "%s"' % (module, e))
     232    except ValueError, e:
     233        raise ImproperlyConfigured('Error importing upload handler module Is FILE_UPLOAD_HANDLERS a correctly defined list or tuple?')
     234    try:
     235        cls = getattr(mod, attr)
     236    except AttributeError:
     237        raise ImproperlyConfigured('Module "%s" does not define a "%s" upload handler backend' % (module, attr))
     238    return cls(*args, **kwargs)
  • django/core/files/move.py

     
     1import os
     2
     3__all__ = ('file_move_safe',)
     4
     5try:
     6    import shutil
     7    file_move = shutil.move
     8except ImportError:
     9    file_move = os.rename
     10
     11def file_move_safe(old_file_name, new_file_name, chunk_size = 1024*64, allow_overwrite=False):
     12    """
     13    Moves a file from one location to another in the safest way possible.
     14   
     15    First, it tries using shutils.move, which is OS-dependent but doesn't
     16    break with change of filesystems. Then it tries os.rename, which will
     17    break if it encounters a change in filesystems. Lastly, it streams
     18    it manually from one file to another in python.
     19
     20    Without ``allow_overwrite``, if the destination file exists, the
     21    file will raise an IOError.
     22    """
     23
     24    from django.core.files import locks
     25
     26    if old_file_name == new_file_name:
     27        # No file moving takes place.
     28        return
     29
     30    if not allow_overwrite and os.path.exists(new_file_name):
     31        raise IOError("Django does not allow overwriting files.")
     32
     33    try:
     34        file_move(old_file_name, new_file_name)
     35        return
     36    except OSError: # moving to another filesystem
     37        pass
     38
     39    new_file = open(new_file_name, 'wb')
     40    # exclusive lock
     41    locks.lock(new_file, locks.LOCK_EX)
     42    old_file = open(old_file_name, 'rb')
     43    current_chunk = None
     44
     45    while current_chunk != '':
     46        current_chunk = old_file.read(chunk_size)
     47        new_file.write(current_chunk)
     48
     49    new_file.close()
     50    old_file.close()
     51
     52    os.remove(old_file_name)
  • django/newforms/fields.py

     
    44
    55import copy
    66import datetime
     7import warnings
    78import os
    89import re
    910import time
     
    416417
    417418class UploadedFile(StrAndUnicode):
    418419    "A wrapper for files uploaded in a FileField"
    419     def __init__(self, filename, content):
     420    def __init__(self, filename, data):
    420421        self.filename = filename
    421         self.content = content
     422        self.data = data
    422423
    423424    def __unicode__(self):
    424425        """
     
    444445            return None
    445446        elif not data and initial:
    446447            return initial
     448
     449        if isinstance(data, dict):
     450            # We warn once, then support both ways below.
     451            warnings.warn("The dictionary usage for files is deprecated. Use the new object interface instead.", DeprecationWarning)
     452
    447453        try:
    448             f = UploadedFile(data['filename'], data['content'])
    449         except TypeError:
     454            file_name = data.file_name
     455            file_size = data.file_size
     456        except AttributeError:
     457            try:
     458                file_name = data.get('filename')
     459                file_size = bool(data['content'])
     460            except (AttributeError, KeyError):
     461                raise ValidationError(self.error_messages['invalid'])
     462
     463        if not file_name:
    450464            raise ValidationError(self.error_messages['invalid'])
    451         except KeyError:
    452             raise ValidationError(self.error_messages['missing'])
    453         if not f.content:
     465        if not file_size:
    454466            raise ValidationError(self.error_messages['empty'])
    455         return f
    456467
     468        return UploadedFile(file_name, data)
     469
    457470class ImageField(FileField):
    458471    default_error_messages = {
    459472        'invalid_image': _(u"Upload a valid image. The file you uploaded was either not an image or a corrupted image."),
     
    470483        elif not data and initial:
    471484            return initial
    472485        from PIL import Image
    473         from cStringIO import StringIO
     486
     487        # We need to get the file, it either has a path
     488        # or we have to read it all into memory...
     489        if hasattr(data, 'temporary_file_path'):
     490            file = data.temporary_file_path()
     491        else:
     492            try:
     493                from cStringIO import StringIO
     494            except ImportError:
     495                from StringIO import StringIO
     496            if hasattr(data, 'read'):
     497                file = StringIO(data.read())
     498            else:
     499                file = StringIO(data['content'])
     500
    474501        try:
    475502            # load() is the only method that can spot a truncated JPEG,
    476503            #  but it cannot be called sanely after verify()
    477             trial_image = Image.open(StringIO(f.content))
     504            trial_image = Image.open(file)
    478505            trial_image.load()
     506
     507            # Since we're about to use the file again, we have to
     508            # reset the cursor of the file object if it has a cursor
     509            # to reset.
     510            if hasattr(file, 'reset'):
     511                file.reset()
     512
    479513            # verify() is the only method that can spot a corrupt PNG,
    480514            #  but it must be called immediately after the constructor
    481             trial_image = Image.open(StringIO(f.content))
     515            trial_image = Image.open(file)
    482516            trial_image.verify()
    483517        except Exception: # Python Imaging Library doesn't recognize it as an image
    484518            raise ValidationError(self.error_messages['invalid_image'])
  • django/utils/datastructures.py

     
    332332            except TypeError: # Special-case if current isn't a dict.
    333333                current = {bits[-1]: v}
    334334
    335 class FileDict(dict):
     335class ImmutableList(tuple):
    336336    """
    337     A dictionary used to hold uploaded file contents. The only special feature
    338     here is that repr() of this object won't dump the entire contents of the
    339     file to the output. A handy safeguard for a large file upload.
     337    A tuple-like object that raises useful
     338    errors when it is asked to mutate.
     339    Example::
     340        a = ImmutableList(range(5), warning=AttributeError("You cannot mutate this."))
     341        a[3] = '4'
     342        (Raises the AttributeError)
    340343    """
    341     def __repr__(self):
    342         if 'content' in self:
    343             d = dict(self, content='<omitted>')
    344             return dict.__repr__(d)
    345         return dict.__repr__(self)
     344
     345    def __new__(cls, *args, **kwargs):
     346        if 'warning' in kwargs:
     347            warning = kwargs['warning']
     348            del kwargs['warning']
     349        else:
     350            warning = 'ImmutableList object is immutable.'
     351        self = tuple.__new__(cls, *args, **kwargs)
     352        self.warning = warning
     353        return self
     354
     355    def complain(self, *wargs, **kwargs):
     356        if isinstance(self.warning, Exception):
     357            raise self.warning
     358        else:
     359            raise AttributeError, self.warning
     360
     361    # All list mutation functions become complain.
     362    __delitem__ = __delslice__ = __iadd__ = __imul__ = complain
     363    __setitem__ = __setslice__ = complain
     364    append = extend = insert = pop = remove = complain
     365    sort = reverse = complain
  • django/utils/text.py

     
    33from django.utils.encoding import force_unicode
    44from django.utils.functional import allow_lazy
    55from django.utils.translation import ugettext_lazy
     6from htmlentitydefs import name2codepoint
    67
    78# Capitalizes the first letter of a string.
    89capfirst = lambda x: x and force_unicode(x)[0].upper() + force_unicode(x)[1:]
     
    218219            yield bit
    219220smart_split = allow_lazy(smart_split, unicode)
    220221
     222def _replace_entity(match):
     223     text = match.group(1)
     224     if text[0] == u'#':
     225         text = text[1:]
     226         try:
     227             if text[0] in u'xX':
     228                 c = int(text[1:], 16)
     229             else:
     230                 c = int(text)
     231             return unichr(c)
     232         except ValueError:
     233             return match.group(0)
     234     else:
     235         try:
     236             return unichr(name2codepoint[text])
     237         except (ValueError, KeyError):
     238             return match.group(0)
     239
     240_entity_re = re.compile(r"&(#?[xX]?(?:[0-9a-fA-F]+|\w{1,8}));")
     241
     242def unescape_entities(text):
     243     return _entity_re.sub(_replace_entity, text)
     244unescape_entities = allow_lazy(unescape_entities, unicode)
  • tests/modeltests/model_forms/models.py

     
    6767
    6868class ImageFile(models.Model):
    6969    description = models.CharField(max_length=20)
    70     image = models.FileField(upload_to=tempfile.gettempdir())
     70    try:
     71        # If PIL is available, try testing PIL.
     72        # Otherwise, it's equivalent to TextFile above.
     73        import Image
     74        image = models.ImageField(upload_to=tempfile.gettempdir())
     75    except ImportError:
     76        image = models.FileField(upload_to=tempfile.gettempdir())
    7177
    7278    def __unicode__(self):
    7379        return self.description
     
    7581__test__ = {'API_TESTS': """
    7682>>> from django import newforms as forms
    7783>>> from django.newforms.models import ModelForm
     84>>> from django.core.files.uploadedfile import SimpleUploadedFile
     85>>> from warnings import filterwarnings
     86>>> filterwarnings("ignore")
    7887
    7988The bare bones, absolutely nothing custom, basic case.
    8089
     
    792801
    793802# Upload a file and ensure it all works as expected.
    794803
     804>>> f = TextFileForm(data={'description': u'Assistance'}, files={'file': SimpleUploadedFile('test1.txt', 'hello world')})
     805>>> f.is_valid()
     806True
     807>>> type(f.cleaned_data['file'])
     808<class 'django.newforms.fields.UploadedFile'>
     809>>> instance = f.save()
     810>>> instance.file
     811u'...test1.txt'
     812
     813>>> os.unlink(instance.get_file_filename())
     814
    795815>>> f = TextFileForm(data={'description': u'Assistance'}, files={'file': {'filename': 'test1.txt', 'content': 'hello world'}})
    796816>>> f.is_valid()
    797817True
     
    814834u'...test1.txt'
    815835
    816836# Delete the current file since this is not done by Django.
    817 
    818837>>> os.unlink(instance.get_file_filename())
    819838
    820839# Override the file by uploading a new one.
    821840
    822 >>> f = TextFileForm(data={'description': u'Assistance'}, files={'file': {'filename': 'test2.txt', 'content': 'hello world'}}, instance=instance)
     841>>> f = TextFileForm(data={'description': u'Assistance'}, files={'file': SimpleUploadedFile('test2.txt', 'hello world')}, instance=instance)
    823842>>> f.is_valid()
    824843True
    825844>>> instance = f.save()
    826845>>> instance.file
    827846u'...test2.txt'
    828847
     848# Delete the current file since this is not done by Django.
     849>>> os.unlink(instance.get_file_filename())
     850
     851>>> f = TextFileForm(data={'description': u'Assistance'}, files={'file': {'filename': 'test2.txt', 'content': 'hello world'}})
     852>>> f.is_valid()
     853True
     854>>> instance = f.save()
     855>>> instance.file
     856u'...test2.txt'
     857
     858# Delete the current file since this is not done by Django.
     859>>> os.unlink(instance.get_file_filename())
     860
    829861>>> instance.delete()
    830862
    831863# Test the non-required FileField
     
    838870>>> instance.file
    839871''
    840872
    841 >>> f = TextFileForm(data={'description': u'Assistance'}, files={'file': {'filename': 'test3.txt', 'content': 'hello world'}}, instance=instance)
     873>>> f = TextFileForm(data={'description': u'Assistance'}, files={'file': SimpleUploadedFile('test3.txt', 'hello world')}, instance=instance)
    842874>>> f.is_valid()
    843875True
    844876>>> instance = f.save()
    845877>>> instance.file
    846878u'...test3.txt'
     879
     880# Delete the current file since this is not done by Django.
     881>>> os.unlink(instance.get_file_filename())
    847882>>> instance.delete()
    848883
     884>>> f = TextFileForm(data={'description': u'Assistance'}, files={'file': {'filename': 'test3.txt', 'content': 'hello world'}})
     885>>> f.is_valid()
     886True
     887>>> instance = f.save()
     888>>> instance.file
     889u'...test3.txt'
     890
     891# Delete the current file since this is not done by Django.
     892>>> os.unlink(instance.get_file_filename())
     893>>> instance.delete()
     894
    849895# ImageField ###################################################################
    850896
    851897# ImageField and FileField are nearly identical, but they differ slighty when
     
    858904
    859905>>> image_data = open(os.path.join(os.path.dirname(__file__), "test.png")).read()
    860906
     907>>> f = ImageFileForm(data={'description': u'An image'}, files={'image': SimpleUploadedFile('test.png', image_data)})
     908>>> f.is_valid()
     909True
     910>>> type(f.cleaned_data['image'])
     911<class 'django.newforms.fields.UploadedFile'>
     912>>> instance = f.save()
     913>>> instance.image
     914u'...test.png'
     915
     916# Delete the current file since this is not done by Django.
     917>>> os.unlink(instance.get_image_filename())
     918
    861919>>> f = ImageFileForm(data={'description': u'An image'}, files={'image': {'filename': 'test.png', 'content': image_data}})
    862920>>> f.is_valid()
    863921True
     
    885943
    886944# Override the file by uploading a new one.
    887945
    888 >>> f = ImageFileForm(data={'description': u'Changed it'}, files={'image': {'filename': 'test2.png', 'content': image_data}}, instance=instance)
     946>>> f = ImageFileForm(data={'description': u'Changed it'}, files={'image': SimpleUploadedFile('test2.png', image_data)}, instance=instance)
    889947>>> f.is_valid()
    890948True
    891949>>> instance = f.save()
    892950>>> instance.image
    893951u'...test2.png'
    894952
     953# Delete the current file since this is not done by Django.
     954>>> os.unlink(instance.get_image_filename())
    895955>>> instance.delete()
    896956
     957>>> f = ImageFileForm(data={'description': u'Changed it'}, files={'image': {'filename': 'test2.png', 'content': image_data}})
     958>>> f.is_valid()
     959True
     960>>> instance = f.save()
     961>>> instance.image
     962u'...test2.png'
     963
     964# Delete the current file since this is not done by Django.
     965>>> os.unlink(instance.get_image_filename())
     966>>> instance.delete()
     967
    897968# Test the non-required ImageField
    898969
    899970>>> f = ImageFileForm(data={'description': u'Test'})
     
    904975>>> instance.image
    905976''
    906977
    907 >>> f = ImageFileForm(data={'description': u'And a final one'}, files={'image': {'filename': 'test3.png', 'content': image_data}}, instance=instance)
     978>>> f = ImageFileForm(data={'description': u'And a final one'}, files={'image': SimpleUploadedFile('test3.png', image_data)}, instance=instance)
    908979>>> f.is_valid()
    909980True
    910981>>> instance = f.save()
    911982>>> instance.image
    912983u'...test3.png'
     984
     985# Delete the current file since this is not done by Django.
     986>>> os.unlink(instance.get_image_filename())
    913987>>> instance.delete()
    914988
     989>>> f = ImageFileForm(data={'description': u'And a final one'}, files={'image': {'filename': 'test3.png', 'content': image_data}})
     990>>> f.is_valid()
     991True
     992>>> instance = f.save()
     993>>> instance.image
     994u'...test3.png'
     995>>> instance.delete()
     996
    915997"""}
  • tests/regressiontests/bug639/tests.py

     
    99from regressiontests.bug639.models import Photo
    1010from django.http import QueryDict
    1111from django.utils.datastructures import MultiValueDict
     12from django.core.files.uploadedfile import SimpleUploadedFile
    1213
    1314class Bug639Test(unittest.TestCase):
    1415       
     
    2122       
    2223        # Fake a request query dict with the file
    2324        qd = QueryDict("title=Testing&image=", mutable=True)
    24         qd["image_file"] = {
    25             "filename" : "test.jpg",
    26             "content-type" : "image/jpeg",
    27             "content" : img
    28         }
    29        
     25        qd["image_file"] = SimpleUploadedFile('test.jpg', img, 'image/jpeg')
     26
    3027        manip = Photo.AddManipulator()
    3128        manip.do_html2python(qd)
    3229        p = manip.save(qd)
     
    3936        Make sure to delete the "uploaded" file to avoid clogging /tmp.
    4037        """
    4138        p = Photo.objects.get()
    42         os.unlink(p.get_image_filename())
    43  No newline at end of file
     39        os.unlink(p.get_image_filename())
  • tests/regressiontests/forms/error_messages.py

     
    11# -*- coding: utf-8 -*-
    22tests = r"""
    33>>> from django.newforms import *
     4>>> from django.core.files.uploadedfile import SimpleUploadedFile
    45
    56# CharField ###################################################################
    67
     
    214215Traceback (most recent call last):
    215216...
    216217ValidationError: [u'INVALID']
    217 >>> f.clean({})
     218>>> f.clean(SimpleUploadedFile('name', None))
    218219Traceback (most recent call last):
    219220...
    220 ValidationError: [u'MISSING']
    221 >>> f.clean({'filename': 'name', 'content':''})
     221ValidationError: [u'EMPTY FILE']
     222>>> f.clean(SimpleUploadedFile('name', ''))
    222223Traceback (most recent call last):
    223224...
    224225ValidationError: [u'EMPTY FILE']
  • tests/regressiontests/forms/tests.py

     
    2626from regressions import tests as regression_tests
    2727from util import tests as util_tests
    2828from widgets import tests as widgets_tests
     29from warnings import filterwarnings
     30filterwarnings("ignore")
    2931
    3032__test__ = {
    3133    'extra_tests': extra_tests,
  • tests/regressiontests/forms/fields.py

     
    22tests = r"""
    33>>> from django.newforms import *
    44>>> from django.newforms.widgets import RadioFieldRenderer
     5>>> from django.core.files.uploadedfile import SimpleUploadedFile
    56>>> import datetime
    67>>> import time
    78>>> import re
     
    773774>>> f.clean({})
    774775Traceback (most recent call last):
    775776...
    776 ValidationError: [u'No file was submitted.']
     777ValidationError: [u'No file was submitted. Check the encoding type on the form.']
    777778
    778779>>> f.clean({}, '')
    779780Traceback (most recent call last):
    780781...
    781 ValidationError: [u'No file was submitted.']
     782ValidationError: [u'No file was submitted. Check the encoding type on the form.']
    782783
    783784>>> f.clean({}, 'files/test3.pdf')
    784785'files/test3.pdf'
     
    788789...
    789790ValidationError: [u'No file was submitted. Check the encoding type on the form.']
    790791
    791 >>> f.clean({'filename': 'name', 'content': None})
     792>>> f.clean(SimpleUploadedFile('name', None))
    792793Traceback (most recent call last):
    793794...
    794795ValidationError: [u'The submitted file is empty.']
    795796
    796 >>> f.clean({'filename': 'name', 'content': ''})
     797>>> f.clean(SimpleUploadedFile('name', ''))
    797798Traceback (most recent call last):
    798799...
    799800ValidationError: [u'The submitted file is empty.']
    800801
    801 >>> type(f.clean({'filename': 'name', 'content': 'Some File Content'}))
     802>>> type(f.clean(SimpleUploadedFile('name', 'Some File Content')))
    802803<class 'django.newforms.fields.UploadedFile'>
    803804
    804 >>> type(f.clean({'filename': 'name', 'content': 'Some File Content'}, 'files/test4.pdf'))
     805>>> type(f.clean(SimpleUploadedFile('name', 'Some File Content'), 'files/test4.pdf'))
    805806<class 'django.newforms.fields.UploadedFile'>
    806807
    807808# URLField ##################################################################
  • tests/regressiontests/forms/forms.py

     
    11# -*- coding: utf-8 -*-
    22tests = r"""
    33>>> from django.newforms import *
     4>>> from django.core.files.uploadedfile import SimpleUploadedFile
    45>>> import datetime
    56>>> import time
    67>>> import re
     
    14651466>>> print f
    14661467<tr><th>File1:</th><td><ul class="errorlist"><li>This field is required.</li></ul><input type="file" name="file1" /></td></tr>
    14671468
    1468 >>> f = FileForm(data={}, files={'file1': {'filename': 'name', 'content':''}}, auto_id=False)
     1469>>> f = FileForm(data={}, files={'file1': SimpleUploadedFile('name', '')}, auto_id=False)
    14691470>>> print f
    14701471<tr><th>File1:</th><td><ul class="errorlist"><li>The submitted file is empty.</li></ul><input type="file" name="file1" /></td></tr>
    14711472
     
    14731474>>> print f
    14741475<tr><th>File1:</th><td><ul class="errorlist"><li>No file was submitted. Check the encoding type on the form.</li></ul><input type="file" name="file1" /></td></tr>
    14751476
    1476 >>> f = FileForm(data={}, files={'file1': {'filename': 'name', 'content':'some content'}}, auto_id=False)
     1477>>> f = FileForm(data={}, files={'file1': SimpleUploadedFile('name', 'some content')}, auto_id=False)
    14771478>>> print f
    14781479<tr><th>File1:</th><td><input type="file" name="file1" /></td></tr>
    14791480>>> f.is_valid()
  • tests/regressiontests/test_client_regress/views.py

     
    11from django.contrib.auth.decorators import login_required
    22from django.http import HttpResponse, HttpResponseRedirect, HttpResponseServerError
     3import sha
    34
    45def no_template_view(request):
    56    "A simple view that expects a GET request, and returns a rendered template"
     
    1011    Check that a file upload can be updated into the POST dictionary without
    1112    going pear-shaped.
    1213    """
     14    from django.core.files.uploadedfile import UploadedFile
    1315    form_data = request.POST.copy()
    1416    form_data.update(request.FILES)
    15     if isinstance(form_data['file_field'], dict) and isinstance(form_data['name'], unicode):
     17    if isinstance(form_data.get('file_field'), UploadedFile) and isinstance(form_data['name'], unicode):
    1618        return HttpResponse('')
    1719    else:
    1820        return HttpResponseServerError()
    1921
     22def file_upload_view_verify(request):
     23    """
     24    Use the sha digest hash to verify the uploaded contents.
     25    """
     26    from django.core.files.uploadedfile import UploadedFile
     27    form_data = request.POST.copy()
     28    form_data.update(request.FILES)
     29
     30    # Check to see if unicode names worked out.
     31    if not request.FILES['file_unicode'].file_name.endswith(u'test_\u4e2d\u6587_Orl\xe9ans.jpg'):
     32        return HttpResponseServerError()
     33
     34    for key, value in form_data.items():
     35        if key.endswith('_hash'):
     36            continue
     37        if key + '_hash' not in form_data:
     38            continue
     39        submitted_hash = form_data[key + '_hash']
     40        if isinstance(value, UploadedFile):
     41            new_hash = sha.new(value.read()).hexdigest()
     42        else:
     43            new_hash = sha.new(value).hexdigest()
     44        if new_hash != submitted_hash:
     45            return HttpResponseServerError()
     46
     47    return HttpResponse('')
     48
    2049def get_view(request):
    2150    "A simple login protected view"
    2251    return HttpResponse("Hello world")
     
    3766def login_protected_redirect_view(request):
    3867    "A view that redirects all requests to the GET view"
    3968    return HttpResponseRedirect('/test_client_regress/get_view/')
    40 login_protected_redirect_view = login_required(login_protected_redirect_view)
    41  No newline at end of file
     69login_protected_redirect_view = login_required(login_protected_redirect_view)
  • tests/regressiontests/test_client_regress/models.py

     
    55from django.test import Client, TestCase
    66from django.core.urlresolvers import reverse
    77import os
     8import sha
    89
    910class AssertContainsTests(TestCase):
    1011    def test_contains(self):
     
    243244        response = self.client.post('/test_client_regress/file_upload/', post_data)
    244245        self.assertEqual(response.status_code, 200)
    245246
     247    def test_large_upload(self):
     248        import tempfile
     249        dir = tempfile.gettempdir()
     250
     251        (fd, name1) = tempfile.mkstemp(suffix='.file1', dir=dir)
     252        file1 = os.fdopen(fd, 'w+b')
     253        file1.write('a' * (2 ** 21))
     254        file1.seek(0)
     255
     256        (fd, name2) = tempfile.mkstemp(suffix='.file2', dir=dir)
     257        file2 = os.fdopen(fd, 'w+b')
     258        file2.write('a' * (10 * 2 ** 20))
     259        file2.seek(0)
     260
     261        # This file contains chinese symbols for a name.
     262        name3 = os.path.join(dir, u'test_&#20013;&#25991;_Orl\u00e9ans.jpg')
     263        file3 = open(name3, 'w+b')
     264        file3.write('b' * (2 ** 10))
     265        file3.seek(0)
     266
     267        post_data = {
     268            'name': 'Ringo',
     269            'file_field1': file1,
     270            'file_field2': file2,
     271            'file_unicode': file3,
     272            }
     273
     274        for key in post_data.keys():
     275            try:
     276                post_data[key + '_hash'] = sha.new(post_data[key].read()).hexdigest()
     277                post_data[key].seek(0)
     278            except AttributeError:
     279                post_data[key + '_hash'] = sha.new(post_data[key]).hexdigest()
     280
     281        response = self.client.post('/test_client_regress/file_upload_verify/', post_data)
     282
     283        for name in (name1, name2, name3):
     284            try:
     285                os.unlink(name)
     286            except:
     287                pass
     288
     289        self.assertEqual(response.status_code, 200)
     290
     291
    246292class LoginTests(TestCase):
    247293    fixtures = ['testdata']
    248294
  • tests/regressiontests/test_client_regress/urls.py

     
    44urlpatterns = patterns('',
    55    (r'^no_template_view/$', views.no_template_view),
    66    (r'^file_upload/$', views.file_upload_view),
     7    (r'^file_upload_verify/$', views.file_upload_view_verify),
    78    (r'^get_view/$', views.get_view),
    89    url(r'^arg_view/(?P<name>.+)/$', views.view_with_argument, name='arg_view'),
    910    (r'^login_protected_redirect_view/$', views.login_protected_redirect_view)
  • tests/regressiontests/datastructures/tests.py

     
    117117>>> d['person']['2']['firstname']
    118118['Adrian']
    119119
    120 ### FileDict ################################################################
    121 
    122 >>> d = FileDict({'content': 'once upon a time...'})
     120### ImmutableList ################################################################
     121>>> d = ImmutableList(range(10))
     122>>> d.sort()
     123Traceback (most recent call last):
     124  File "<stdin>", line 1, in <module>
     125  File "/var/lib/python-support/python2.5/django/utils/datastructures.py", line 359, in complain
     126    raise AttributeError, self.warning
     127AttributeError: ImmutableList object is immutable.
    123128>>> repr(d)
    124 "{'content': '<omitted>'}"
    125 >>> d = FileDict({'other-key': 'once upon a time...'})
    126 >>> repr(d)
    127 "{'other-key': 'once upon a time...'}"
     129'(0, 1, 2, 3, 4, 5, 6, 7, 8, 9)'
     130>>> d = ImmutableList(range(10), warning="Object is immutable!")
     131>>> d[1]
     1321
     133>>> d[1] = 'test'
     134Traceback (most recent call last):
     135  File "<stdin>", line 1, in <module>
     136  File "/var/lib/python-support/python2.5/django/utils/datastructures.py", line 359, in complain
     137    raise AttributeError, self.warning
     138AttributeError: Object is immutable!
    128139"""
  • AUTHORS

     
    5858    Jökull Sólberg Auðunsson <jokullsolberg@gmail.com>
    5959    Arthur <avandorp@gmail.com>
    6060    David Avsajanishvili <avsd05@gmail.com>
    61     axiak@mit.edu
     61    Mike Axiak <axiak@mit.edu>
    6262    Niran Babalola <niran@niran.org>
    6363    Morten Bagai <m@bagai.com>
    6464    Mikaël Barbero <mikael.barbero nospam at nospam free.fr>
     
    136136    Marc Fargas <telenieko@telenieko.com>
    137137    Szilveszter Farkas <szilveszter.farkas@gmail.com>
    138138    favo@exoweb.net
     139    fdr <drfarina@gmail.com>
    139140    Dmitri Fedortchenko <zeraien@gmail.com>
    140141    Liang Feng <hutuworm@gmail.com>
    141142    Bill Fenner <fenner@gmail.com>
  • docs/upload_handling.txt

     
     1============
     2File Uploads
     3============
     4
     5**New in Django development version**
     6
     7Most Web sites wouldn't be complete without a way to upload files. Before the
     8file even gets into ``request.FILES`` Django has to decide where to put the
     9all the incoming data. This document describes how Django takes the incoming
     10request and populates ``request.FILES``.
     11
     12.. _Request and response objects: ../request_response/#attributes
     13
     14Default Behavior
     15================
     16
     17After setting up a quick model that contains a `FileField`_ and enabling the
     18`Admin interface`_ you can quickly use a form to upload files to your site.
     19By default, if the upload is smaller than **2.5 Megabytes** in size, Django
     20will hold the entire contents of the upload in memory, and the file will
     21quickly be saved to its final destination (as defined by
     22`settings.MEDIA_ROOT`_) without any intermediary.
     23
     24If the entire upload is larger than 2.5 Megabytes, then it will -- by
     25default -- write the contents of the uploaded file to a temporary file
     26in your operating system's default temporary directory. On a posix platform,
     27this means that you could expect Django to generate a file similar in name
     28to ``/tmp/tmpzfp6I6.upload``. During the upload, you may notice this file
     29grow in size as Django loads the data onto disk.
     30
     31.. note::
     32    You may find that the default temporary directory is not writable.
     33    This is especially true if you are on shared hosting. If this is
     34    the case, Django will raise an error when you try to upload. Please
     35    read below in `Extending the Default`_ to change this directory.
     36
     37
     38.. _FileField: ../model-api/#filefield
     39.. _Admin interface: ../tutorial02/#activate-the-admin-site
     40.. _settings.MEDIA_ROOT: ../settings/#media-root
     41
     42Extending the Default
     43=====================
     44
     45Suppose you have quite a bit of memory. You may decide that you want Django
     46to stream only if the entire upload exceeds 10 Megabytes. You may even
     47decide that whatever directory is the platform default is not suitable
     48for your project. If this is the case, Django provides these two settings:
     49
     50    =============================== ======================================
     51    Setting                         Description
     52    =============================== ======================================
     53    ``FILE_UPLOAD_MAX_MEMORY_SIZE`` The maximum size of a request in bytes
     54                                    for which Django will try to load the
     55                                    entire upload contents in memory.
     56
     57    ``FILE_UPLOAD_TEMP_DIR``        The directory on the file system where
     58                                    uploaded contents will be temporarily
     59                                    stored if not completely in memory.
     60                                    E.g.: ``"/tmp"``
     61    =============================== ======================================
     62
     63There is one final setting -- ``FILE_UPLOAD_HANDLERS`` -- which allows complete
     64customization of the upload process.
     65
     66Upload Handlers
     67===============
     68
     69Through upload handlers Django provides the flexibility to extend the
     70upload process beyond simple storage. You can use custom handlers to enforce
     71user-level quotas, compress data on the fly, render progress bars, and
     72even send data to another warehouse directly without storing it locally.
     73
     74There are two pieces to the Django upload handling: the upload handler and the
     75uploaded file. These are both represented by python classes --
     76``FileUploadHandler`` and ``UploadedFile`` respectively. Throughout the lifetime
     77of the upload process, Django will call on the upload handler to handle the
     78upload, while the upload handler is expected to provide Django an ``UploadedFile``
     79object to successfully complete a file's upload.
     80
     81Setting Default Upload Handlers for your Project
     82------------------------------------------------
     83
     84Similar to `Middleware`_, upload handlers have an order that is initially
     85defined in ``settings.FILE_UPLOAD_HANDLERS``. The default value is::
     86
     87    ("django.core.files.fileuploadhandler.MemoryFileUploadHandler",
     88     "django.core.files.fileuploadhandler.TemporaryFileUploadHandler",)
     89
     90This literally means: Try putting the upload in memory first and failing
     91that put the upload in a temporary file.
     92
     93This behavior, however, is completely dependent on how each of those handlers
     94is written. For example, if someone gave you a ``Rot13UploadHandler``, you can
     95update your ``settings`` to contain::
     96
     97   FILE_UPLOAD_HANDLERS = (
     98     "app.uploadhandlers.rot13.Rot13UploadHandler",
     99     "django.core.files.fileuploadhandler.MemoryFileUploadHandler",
     100     "django.core.files.fileuploadhandler.TemporaryFileUploadHandler",
     101   )
     102
     103And the ``Rot13UploadHandler`` will perform the ``rot13`` operation on all
     104data before the data get to subsequent handlers.
     105
     106.. _Middleware: ../middleware/
     107
     108Modifying your Upload Handlers Dynamically
     109------------------------------------------
     110
     111During the lifetime of your project, you may realize that a particular
     112view or views require different uploading behavior. For this reason,
     113the ``request`` object contains a list of handlers
     114(``request.upload_handlers``) that will be called in order. To append
     115a handler to the list, you would append to it like any other list.
     116For example, suppose you had an ``ProgressBarUploadHandler`` class.
     117To append it to your upload handlers you would write::
     118
     119    request.upload_handlers.append(ProgressBarUploadHandler())
     120
     121However, since the progress bar handler would probably need to run before the
     122other handlers get a chance, you'd probably want to insert it. That is, you'd
     123write::
     124
     125   request.upload_handlers.insert(0, ProgressBarUploadHandler())
     126
     127If you want to replace the upload handlers completely, you can just assign a new
     128list::
     129
     130   request.upload_handlers = [ProgressBarUploadHandler()]
     131
     132And all Django will do is keep a progress, but do nothing with the file itself!
     133One more note: After the upload has completed, you are no longer allowed to
     134assign or modify this list, and Django will raise an error if you try to modify
     135this list after an upload.
     136
     137Writing a File Upload Handler
     138-----------------------------
     139
     140All file upload handlers are subclasses of ``FileUploadHandler``, found in
     141``django.core.files.fileuploadhandler``. To create your handler, you need to
     142define the required methods followed by the methods that make most sense to you:
     143
     144chunk_size
     145~~~~~~~~~~
     146
     147An integer attribute that specifies what sized chunks Django should store
     148into memory and feed into the handler. The chunk sizes should be divisible by
     149``4`` and should not exceed ``2 ** 31 - 1`` in size. When there are multiple
     150chunk sizes provided by multiple handlers, Django will use the smallest chunk
     151size.
     152
     153__init__
     154~~~~~~~~
     155
     156The constructor, which should optionally support taking a request object as its
     157first argument. When called via ``settings.FILE_UPLOAD_HANDLERS``, it will be
     158passed a request object.
     159
     160Interface: ``__init__(self, request=None)``
     161
     162``request`` is the request object. Do not access ``request.POST`` or
     163``request.FILES``.
     164
     165new_file
     166~~~~~~~~
     167
     168``new_file`` signals that a new file is starting. You can initialize a file or
     169whatever else is needed on a new file upload.
     170
     171Interface: ``new_file(self, field_name, file_name, content_type, content_length,
     172charset)``
     173
     174``field_name`` is a string name of the field this was POSTed as.
     175
     176``file_name`` is the unicode filename that was provided by the browser.
     177
     178``content_type`` is the MIME type provided by the browser -- E.g.
     179``'image/jpeg'``.
     180
     181``content_length`` is the length of the image given by the browser if provided,
     182``None`` otherwise.
     183
     184``charset`` is the charset given by the browser if provided, ``None`` otherwise.
     185
     186Returns: ``None`` if you want other handlers to get ``new_file`` called.
     187Something nonzero if you don't want subsequent handlers to get a chance.
     188
     189receive_data_chunk
     190~~~~~~~~~~~~~~~~~~
     191*required*
     192
     193Receives a segment of data from the file upload. For example: The
     194``TemporaryFileUploadHandler`` takes the data and writes it to disk.
     195
     196Interface: ``receive_data_chunk(self, raw_data, start)``
     197
     198``raw_data`` is a byte string containing the uploaded data.
     199
     200``start`` is the position in the file where this ``raw_data`` chunk begins.
     201
     202Returns: ``None`` if you don't want the subsequent upload handlers to receive
     203the data. Whatever else you return gets fed into the subsequent upload handlers'
     204``receive_data_chunk`` method. In this way, one handler can be a "filter" for
     205other handlers.
     206
     207Exceptions: If you raise a ``StopUpload`` or a ``SkipFile`` exception, the
     208upload will abort or the file will be skipped respectively.
     209
     210file_complete
     211~~~~~~~~~~~~~
     212*required*
     213
     214Signals that a file has finished uploading. Expected to return an
     215``UploadedFile`` object if the file has been packaged successfully.
     216
     217Interface: ``file_complete(self, file_size)``
     218
     219``file_size`` is the number of bytes you have received for this file.
     220
     221Returns: An ``UploadedFile`` object to set in the ``request.FILES`` dictionary.
     222``None`` if you want the subsequent upload handlers to get an ``UploadedFile``
     223object.
     224
     225upload_complete
     226~~~~~~~~~~~~~~~
     227
     228Defines when the entire upload has completed.
     229
     230Interface: ``upload_complete(self)``
     231
     232handle_raw_input
     233~~~~~~~~~~~~~~~~
     234
     235Allows the handler to completely override the parsing of the raw HTTP-layered
     236input.
     237
     238Interface: ``handle_raw_input(self, input_data, META, content_length, boundary,
     239encoding)``
     240
     241``input_data`` is a file-like object that supports the read operation.
     242
     243``META`` is the same object as ``request.META``.
     244
     245``content_length`` is the length of the data in ``input_data``. Don't read more
     246than ``content_length`` bytes from ``input_data``.
     247
     248``boundary`` is the MIME boundary for this request.
     249
     250``encoding`` is the encoding of the request.
     251
     252Returns: ``None`` if you want to go on to the next stage, ``(POST, FILES)`` if
     253you want to return the new data structures suitable for the request directly.
     254
     255Defining an Uploaded File
     256-------------------------
     257
     258All file upload handlers are subclasses of ``UploadedFile``, found in
     259``django.core.files.uploadedfile``. The uploaded file object is returned by the
     260handler above in ``file_complete``. To create your own uploaded file class, you
     261need to define the required methods followed by the methods that make most sense
     262to you:
     263
     264read
     265~~~~
     266*required*
     267
     268Interface: ``read(self, num_bytes=None)``
     269
     270Returns: A byte string of length ``num_bytes`` (or the size of the file if
     271``num_bytes`` is not supplied).
     272
     273chunk
     274~~~~~
     275
     276A generator to yield small chunks from the file. With the ``read()`` defined,
     277the ``UploadedFile`` class already defines a ``chunk()`` method that's probably
     278suitable.
     279
     280Interface: ``chunk(self, chunk_size=None)``
     281
     282multiple_chunks
     283~~~~~~~~~~~~~~~
     284
     285Interface: ``multiple_chunks(self, chunk_size=None)``
     286
     287Returns: ``True`` or ``False`` depending on whether or not the user of this
     288file can expect more than one chunk when calling ``chunk(self, chunk_size)``.
     289If all the data is in memory, you should return ``False``.
     290
     291temporary_file_path
     292~~~~~~~~~~~~~~~~~~~
     293
     294If defined, this method should return the file path of the file on the
     295operating system. This will let users move files rather than write to
     296new files if the option is available.
     297
     298Interface: ``temporary_file_path(self)``
     299
     300Returns: A file path in a file system on the local computer.
     301
     302upload_errors
     303~~~~~~~~~~~~~
     304
     305If defined, this method should return a string describing errors that
     306occured during uploads.
     307
     308Interface: ``upload_errors(self)``
     309
     310Returns: A ``unicode`` string describing an error that occured with the
     311file's upload.
  • docs/request_response.txt

     
    8080    strings.
    8181
    8282``FILES``
     83   **New in Django development version**
    8384    A dictionary-like object containing all uploaded files. Each key in
    8485    ``FILES`` is the ``name`` from the ``<input type="file" name="" />``. Each
    85     value in ``FILES`` is a standard Python dictionary with the following three
    86     keys:
     86    value in ``FILES`` is an ``UploadedFile`` object containing at least the
     87    following attributes:
    8788
    88         * ``filename`` -- The name of the uploaded file, as a Python string.
    89         * ``content-type`` -- The content type of the uploaded file.
    90         * ``content`` -- The raw content of the uploaded file.
     89        * ``read(num_bytes=None)`` -- Read a number of bytes from the file.
     90        * ``file_name`` -- The name of the uploaded file.
     91        * ``file_size`` -- The size, in bytes, of the uploaded file.
     92        * ``chunk()`` -- A generator that yields sequential chunks of data.
    9193
    92     Note that ``FILES`` will only contain data if the request method was POST
    93     and the ``<form>`` that posted to the request had
    94     ``enctype="multipart/form-data"``. Otherwise, ``FILES`` will be a blank
    95     dictionary-like object.
     94    See `File Uploads`_ for more information. Note that ``FILES`` will only
     95    contain data if the request method was POST and the ``<form>`` that posted
     96    to the request had ``enctype="multipart/form-data"``. Otherwise, ``FILES``
     97    will be a blank dictionary-like object.
    9698
     99    .. _File Uploads: ../upload_handling/
     100
    97101``META``
    98102    A standard Python dictionary containing all available HTTP headers.
    99103    Available headers depend on the client and server, but here are some
  • docs/settings.txt

     
    279279
    280280The database backend to use. The build-in database backends are
    281281``'postgresql_psycopg2'``, ``'postgresql'``, ``'mysql'``, ``'mysql_old'``,
    282 ``'sqlite3'`` and ``'oracle'``.
     282``'sqlite3'``, ``'oracle'``, and ``'oracle'``.
    283283
    284284In the Django development version, you can use a database backend that doesn't
    285285ship with Django by setting ``DATABASE_ENGINE`` to a fully-qualified path (i.e.
     
    513513The character encoding used to decode any files read from disk. This includes
    514514template files and initial SQL data files.
    515515
     516FILE_UPLOAD_HANDLERS
     517--------------------
     518
     519**New in Django development version**
     520
     521Default::
     522
     523    ("django.core.files.fileuploadhandler.MemoryFileUploadHandler",
     524     "django.core.files.fileuploadhandler.TemporaryFileUploadHandler",)
     525
     526A tuple of handlers to use for uploading.
     527
     528FILE_UPLOAD_MAX_MEMORY_SIZE
     529---------------------------
     530
     531**New in Django development version**
     532
     533Default: ``2621440``
     534
     535The maximum size (in bytes) that an upload will be before it gets streamed to the file system.
     536
     537FILE_UPLOAD_TEMP_DIR
     538--------------------
     539
     540**New in Django development version**
     541
     542Default: ``None``
     543
     544The directory to store data temporarily while uploading files. If ``None``, Django will use the standard temporary directory for the operating system. For example, this will default to '/tmp' on *nix-style operating systems.
     545
    516546FIXTURE_DIRS
    517547-------------
    518548
  • docs/newforms.txt

     
    805805need to bind the file data containing the mugshot image::
    806806
    807807    # Bound form with an image field
     808    >>> from django.core.files.uploadedfile import SimpleUploadedFile
    808809    >>> data = {'subject': 'hello',
    809810    ...         'message': 'Hi there',
    810811    ...         'sender': 'foo@example.com',
    811812    ...         'cc_myself': True}
    812     >>> file_data = {'mugshot': {'filename':'face.jpg'
    813     ...                          'content': <file data>}}
     813    >>> file_data = {'mugshot': SimpleUploadedFile('face.jpg', <file data>)}
    814814    >>> f = ContactFormWithMugshot(data, file_data)
    815815
    816816In practice, you will usually specify ``request.FILES`` as the source
Back to Top