Ticket #2070: 2070_revision7377.diff

File 2070_revision7377.diff, 95.3 KB (added by axiak, 7 years ago)

New 2070 patch...includes fixes to the parser in addition to the newforms fix caught by EricB as well as better docs.

  • django/http/multipartparser.py

     
     1"""
     2MultiPart parsing for file uploads.
     3
     4This object will take the file upload headers
     5and the file upload handler and chunk the upload
     6data for the handler to deal with.
     7"""
     8from django.utils.datastructures import MultiValueDict
     9from django.utils.encoding import force_unicode
     10from django.utils.text import unescape_entities
     11
     12__all__ = ('MultiPartParser','MultiPartParserError','InputStreamExhausted')
     13
     14class MultiPartParserError(Exception):
     15    pass
     16
     17class InputStreamExhausted(Exception):
     18    """ No more reads are allowed from this device. """
     19    pass
     20
     21class FILE(object):
     22    """
     23    Used to denote FILE sections of the multipart from the
     24    ParseBoundaryStream.
     25
     26    FILE sections have a filename field, and are typically used when
     27    the user is uploading a file.
     28    """
     29    pass
     30
     31class RAW(object):
     32    """
     33    Used to denote RAW sections of the multipart from the
     34    ParseBoundaryStream.
     35
     36    RAW is a special marker for a section that doesn't appear to have
     37    headers that allow us to understand its contents. It is not a
     38    standard terminology of RFC2388.
     39    """
     40    pass
     41
     42class FIELD(object):
     43    """
     44    Used to denote RAW sections of the multipart from the
     45    ParseBoundaryStream.
     46
     47    FIELDS are standard post fields, these are typically entries on a
     48    form.
     49    """
     50    pass
     51
     52class MultiPartParser(object):
     53    """
     54    A rfc2388 multipart/form-data parser.
     55
     56    parse() reads the input stream in chunk_size chunks and returns a
     57    tuple of (POST MultiValueDict, FILES MultiValueDict). If
     58    file_upload_dir is defined files will be streamed to temporary
     59    files in the specified directory.
     60    """
     61    def __init__(self, META, input_data, upload_handlers, encoding=None):
     62        """
     63        Initialize the MultiPartParser object.
     64
     65        *META* -- The standard META dictionary in Django request objects.
     66        *input_data* -- The raw post data, as a bytestring.
     67        *upload_handler* -- An object of type UploadHandler
     68                            that performs operations on the uploaded
     69                            data.
     70        *encoding* -- The encoding with which to treat the incoming data.
     71        """
     72        # Import cgi utilities for (near) future use.
     73        global valid_boundary, settings
     74        from django.conf import settings
     75        from cgi import valid_boundary
     76
     77        #
     78        # Content-Type should containt multipart and the boundary information.
     79        #
     80
     81        content_type = META.get('HTTP_CONTENT_TYPE', META.get('CONTENT_TYPE', ''))
     82        if not content_type.startswith('multipart/'):
     83            raise MultiPartParserError('Invalid Content-Type: %s' %
     84                                       content_type)
     85
     86        # Parse the header to get the boundary to split the parts.
     87        ctypes, opts = parse_header(content_type)
     88        boundary = opts.get('boundary')
     89        if not boundary or not valid_boundary(boundary):
     90            raise MultiPartParserError('Invalid boundary in multipart: %s' %
     91                                       boundary)
     92
     93
     94        #
     95        # Content-Length should contain the length of the body we are about
     96        # to receive.
     97        #
     98        try:
     99            content_length = int(META.get('HTTP_CONTENT_LENGTH',
     100                                          META.get('CONTENT_LENGTH',0)))
     101        except (ValueError, TypeError):
     102            # For now set it to 0...we'll try again later on down.
     103            content_length = 0
     104
     105        if content_length <= 0:
     106            # This means we shouldn't continue...raise an error.
     107            raise MultiPartParserError("Invalid content length: %r" % content_length)
     108
     109        self._boundary = boundary
     110        self._input_data = input_data
     111
     112        # For compatibility with low-level network APIs (with 32-bit integers),
     113        # the chunk size should be < 2^31, but still divisible by 4.
     114        self._chunk_size = min(2147483644, *[x.chunk_size for x in upload_handlers
     115                                            if x.chunk_size])
     116
     117        self._meta = META
     118        self._encoding = encoding or settings.DEFAULT_CHARSET
     119        self._content_length = content_length
     120        self._upload_handlers = upload_handlers
     121
     122    def parse(self):
     123        """
     124        Parse the POST data and break it into a FILES MultiValueDict
     125        and a POST MultiValueDict.
     126
     127           *returns* -- A tuple containing the POST and FILES dictionary,
     128                        respectively.
     129        """
     130        from django.core.files.fileuploadhandler import StopUpload, SkipFile
     131        from django.http import QueryDict
     132
     133        encoding = self._encoding
     134        handlers = self._upload_handlers
     135
     136        limited_input_data = LimitBytes(self._input_data, self._content_length)
     137
     138        # See if the handler will want to take care of the parsing.
     139        # This allows overriding everything if somebody wants it.
     140        for handler in handlers:
     141            result = handler.handle_raw_input(limited_input_data,
     142                                              self._meta,
     143                                              self._content_length,
     144                                              self._boundary,
     145                                              encoding)
     146            if result is not None:
     147                return result[0], result[1]
     148
     149        # Create the data structures to be used later.
     150        self._post = QueryDict('', mutable=True)
     151        self._files = MultiValueDict()
     152
     153        # Instantiate the parser and stream:
     154        stream = LazyStream(ChunkIter(limited_input_data, self._chunk_size))
     155
     156        # Whether or not to signal a file-completion at the beginning of the loop.
     157        old_field_name = None
     158        counters = [0] * len(handlers)
     159
     160        for item_type, meta_data, stream in Parser(stream, self._boundary):
     161            if old_field_name:
     162                # We run this at the beginning of the next loop
     163                # since we cannot be sure a file is complete until
     164                # we hit the next boundary/part of the multipart content.
     165                self.handle_file_complete(old_field_name, counters)
     166
     167            try:
     168                disposition = meta_data['content-disposition'][1]
     169                field_name = disposition['name'].strip()
     170            except (KeyError, IndexError, AttributeError):
     171                continue
     172
     173            transfer_encoding = meta_data.get('content-transfer-encoding')
     174            field_name = force_unicode(field_name, encoding, errors='replace')
     175
     176            if item_type is FIELD:
     177                # This is a post field, we can just set it in the post
     178                if transfer_encoding == 'base64':
     179                    raw_data = stream.read()
     180                    try:
     181                        data = str(raw_data).decode('base64')
     182                    except:
     183                        data = raw_data
     184                else:
     185                    data = stream.read()
     186
     187                self._post.appendlist(field_name,
     188                                      force_unicode(data, encoding, errors='replace'))
     189            elif item_type is FILE:
     190                # This is a file, use the handler...
     191                file_successful = True
     192                file_name = disposition.get('filename')
     193                if not file_name:
     194                    continue
     195                file_name = force_unicode(file_name, encoding, errors='replace')
     196                file_name = self.IE_sanitize(unescape_entities(file_name))
     197
     198
     199                content_type = meta_data.get('content-type', ('',))[0].strip()
     200                try:
     201                    charset = meta_data.get('content-type', (0,{}))[1].get('charset', None)
     202                except:
     203                    charset = None
     204
     205                try:
     206                    content_length = int(meta_data.get('content-length')[0])
     207                except (IndexError, TypeError, ValueError):
     208                    content_length = None
     209
     210                counters = [0] * len(handlers)
     211                try:
     212                    for handler in handlers:
     213                        retval = handler.new_file(field_name, file_name,
     214                                                  content_type, content_length,
     215                                                  charset)
     216                        if retval:
     217                            break
     218
     219                    for chunk in stream:
     220                        if transfer_encoding == 'base64':
     221                            # We only special-case base64 transfer encoding
     222                            try:
     223                                chunk = str(chunk).decode('base64')
     224                            except Exception, e:
     225                                # Since this is only a chunk, any error is an unfixable error.
     226                                raise MultiValueParseError("Could not decode base64 data: %r" % e)
     227
     228                        for i, handler in enumerate(handlers):
     229                            chunk_length = len(chunk)
     230                            counters[i] += chunk_length
     231                            chunk = handler.receive_data_chunk(chunk,
     232                                                               counters[i] - chunk_length,
     233                                                               counters[i])
     234                            if chunk is None:
     235                                # If the chunk received by the handler is None, then don't continue.
     236                                break
     237
     238                except (StopUpload, SkipFile), e:
     239                    file_successful = False
     240                    if isinstance(e, SkipFile):
     241                        # Just use up the rest of this file...
     242                        exhaust(stream)
     243                    elif isinstance(e, StopUpload):
     244                        # Abort the parsing and break
     245                        parser.abort()
     246                        break
     247                else:
     248                    # Handle file upload completions on next iteration.
     249                    old_field_name = field_name
     250            else:
     251                # If this is neither a FIELD or a FILE, just exhaust the stream.
     252                exhaust(stream)
     253
     254        # Make sure that the request data is all fed
     255        exhaust(limited_input_data)
     256        # Signal that the upload has completed.
     257        for handler in handlers:
     258            retval = handler.upload_complete()
     259            if retval:
     260                break
     261
     262        return self._post, self._files
     263
     264    def handle_file_complete(self, old_field_name, counters):
     265        """
     266        Handle all the signalling that takes place when a file is complete.
     267        """
     268        for i, handler in enumerate(self._upload_handlers):
     269            file_obj = handler.file_complete(counters[i])
     270            if file_obj:
     271                # If it returns a file object, then set the files dict.
     272                self._files.appendlist(force_unicode(old_field_name,
     273                                                     self._encoding,
     274                                                     errors='replace'),
     275                                       file_obj)
     276                break
     277
     278    def IE_sanitize(self, filename):
     279        """cleanup filename from IE full paths"""
     280        return filename and filename[filename.rfind("\\")+1:].strip()
     281
     282
     283class LazyStream(object):
     284    """
     285    The LazyStream wrapper allows one to pull and "unget" bytes from a stream.
     286
     287    Given a producer object (an iterator that yields bytestrings), the
     288    LazyStream object will support iteration, reading, and keeping a
     289    "look-back" variable in case you need to "unget" some bytes.
     290    """
     291    def __init__(self, producer, length=None):
     292        """
     293        Every LazyStream must have a producer when instantiated.
     294
     295        A producer is an iterable that returns a string each time it
     296        is called.
     297        """
     298        self._producer = producer
     299        self._empty = False
     300        self._leftover = ''
     301        self.length = length
     302        self.position = 0
     303        self._remaining = length
     304
     305    def tell(self):
     306        return self.position
     307
     308    def read(self, size=None):
     309        def parts():
     310            remaining = (size is not None and [size] or [self._remaining])[0]
     311            # do the whole thing in one shot if no limit was provided.
     312            if remaining is None:
     313                yield ''.join(self)
     314                return
     315
     316            # otherwise do some bookkeeping to return exactly enough
     317            # of the stream and stashing any extra content we get from
     318            # the producer
     319            while remaining != 0:
     320                assert remaining > 0, 'remaining bytes to read should never go negative'
     321
     322                chunk = self.next()
     323
     324                emitting = chunk[:remaining]
     325                self.unget(chunk[remaining:])
     326                remaining -= len(emitting)
     327                yield emitting
     328
     329        out = ''.join(parts())
     330        self.position += len(out)
     331        return out
     332
     333    def next(self):
     334        """
     335        Used when the exact number of bytes to read is unimportant.
     336
     337        This procedure just returns whatever is chunk is conveniently
     338        returned from the iterator instead. Useful to avoid
     339        unnecessary bookkeeping if performance is an issue.
     340        """
     341        if self._leftover:
     342            output = self._leftover
     343            self.position += len(output)
     344            self._leftover = ''
     345            return output
     346        else:
     347            output = self._producer.next()
     348            self.position += len(output)
     349            return output
     350
     351    def close(self):
     352        """
     353        Used to invalidate/disable this lazy stream.
     354
     355        Replaces the producer with an empty list. Any leftover bytes
     356        that have already been read will still be reported upon read()
     357        and/or next().
     358        """
     359        self._producer = []
     360
     361    def __iter__(self):
     362        return self
     363
     364    def unget(self, bytes):
     365        """
     366        Places bytes back onto the front of the lazy stream.
     367
     368        Future calls to read() will return those bytes first. The
     369        stream position and thus tell() will be rewound.
     370        """
     371        self.position -= len(bytes)
     372        self._leftover = ''.join([bytes, self._leftover])
     373
     374class ChunkIter(object):
     375    """
     376    An iterable that will yield chunks of data.
     377    Given a file-like object as the constructor,
     378    this object will yield chunks of read operations
     379    from that object.
     380    """
     381    def __init__(self, flo, chunk_size=64 * 1024):
     382        self.flo = flo
     383        self.chunk_size = chunk_size
     384
     385    def next(self):
     386        try:
     387            data = self.flo.read(self.chunk_size)
     388        except InputStreamExhausted:
     389            raise StopIteration
     390        if data:
     391            return data
     392        else:
     393            raise StopIteration
     394
     395    def __iter__(self):
     396        return self
     397
     398
     399class LimitBytes(object):
     400    """ Limit bytes for a file object. """
     401    def __init__(self, fileobject, length):
     402        self._file = fileobject
     403        self.remaining = length
     404
     405    def read(self, num_bytes=None):
     406        """
     407        Read data from the underlying file.
     408        If you ask for too much or there isn't anything left,
     409        this will raise an InputStreamExhausted error.
     410        """
     411        if self.remaining <= 0:
     412            raise InputStreamExhausted()
     413        if num_bytes is None:
     414            num_bytes = self.remaining
     415        else:
     416            num_bytes = min(num_bytes, self.remaining)
     417        self.remaining -= num_bytes
     418        return self._file.read(num_bytes)
     419
     420class InterBoundaryIter(object):
     421    """
     422    A Producer that will iterate over boundaries.
     423    """
     424    def __init__(self, stream, boundary):
     425        self._stream = stream
     426        self._boundary = boundary
     427
     428    def __iter__(self):
     429        return self
     430
     431    def next(self):
     432        try:
     433            return LazyStream(BoundaryIter(self._stream, self._boundary))
     434        except InputStreamExhausted:
     435            raise StopIteration
     436
     437
     438class BoundaryIter(object):
     439    """
     440    A Producer that is sensitive to boundaries.
     441
     442    Will happily yield bytes until a boundary is found. Will yield the
     443    bytes before the boundary, throw away the boundary bytes
     444    themselves, and push the post-boundary bytes back on the stream.
     445
     446    The future calls to .next() after locating the boundary will raise
     447    a StopIteration exception.
     448    """
     449
     450    def __init__(self, stream, boundary):
     451        self._stream = stream
     452        self._boundary = boundary
     453        self._done = False
     454        # rollback an additional six bytes because the format is like
     455        # this: CRLF<boundary>[--CRLF]
     456        self._rollback = len(boundary) + 6
     457
     458        # Try to use mx fast string search if available. Otherwise
     459        # use Python find. Wrap the latter for consistency.
     460        unused_char = self._stream.read(1)
     461        if not unused_char:
     462            raise InputStreamExhausted
     463        self._stream.unget(unused_char)
     464        try:
     465            from mx.TextTools import FS
     466            self._fs = FS(boundary).find
     467        except ImportError:
     468            self._fs = lambda data: data.find(boundary)
     469
     470    def __iter__(self):
     471        return self
     472
     473    def next(self):
     474        if self._done:
     475            raise StopIteration
     476
     477        stream = self._stream
     478        rollback = self._rollback
     479
     480        bytes_read = 0
     481        chunks = []
     482        for bytes in stream:
     483            bytes_read += len(bytes)
     484            chunks.append(bytes)
     485            if bytes_read > rollback:
     486                break
     487            if not bytes:
     488                break
     489        else:
     490            self._done = True
     491
     492        if not chunks:
     493            raise StopIteration
     494
     495        chunk = ''.join(chunks)
     496        boundary = self._find_boundary(chunk, len(chunk) < self._rollback)
     497
     498        if boundary:
     499            end, next = boundary
     500            stream.unget(chunk[next:])
     501            self._done = True
     502            return chunk[:end]
     503        else:
     504            # make sure we dont treat a partial boundary (and
     505            # its separators) as data
     506            if not chunk[:-rollback]:# and len(chunk) >= (len(self._boundary) + 6):
     507                # There's nothing left, we should just return and mark as done.
     508                self._done = True
     509                return chunk
     510            else:
     511                stream.unget(chunk[-rollback:])
     512                return chunk[:-rollback]
     513
     514    def _find_boundary(self, data, eof = False):
     515        """
     516        Finds a multipart boundary in data.
     517
     518        Should no boundry exist in the data None is returned
     519        instead. Otherwise a tuple containing
     520        the indices of the following are returned:
     521
     522         * the end of current encapsulation
     523
     524         * the start of the next encapsulation
     525        """
     526        index = self._fs(data)
     527        if index < 0:
     528            return None
     529        else:
     530            end = index
     531            next = index + len(self._boundary)
     532            data_len = len(data) - 1
     533            # backup over CRLF
     534            if data[max(0,end-1)] == '\n':
     535                end -= 1
     536            if data[max(0,end-1)] == '\r':
     537                end -= 1
     538            # skip over --CRLF
     539            #if data[min(data_len,next)] == '-':
     540            #    next += 1
     541            #if data[min(data_len,next)] == '-':
     542            #    next += 1
     543            #if data[min(data_len,next)] == '\r':
     544            #    next += 1
     545            #if data[min(data_len,next)] == '\n':
     546            #    next += 1
     547            return end, next
     548
     549def exhaust(stream_or_iterable):
     550    """
     551    Completely exhausts an iterator or stream.
     552
     553    Raise a MultiPartParserError if the argument is not a stream or an
     554    iterable.
     555    """
     556    iterator = None
     557    try:
     558        iterator = iter(stream_or_iterable)
     559    except TypeError:
     560        iterator = ChunkIter(stream_or_iterable, 16384)
     561
     562    if iterator is None:
     563        raise MultiPartParserError('multipartparser.exhaust() was passed a non-iterable or stream parameter')
     564
     565    for __ in iterator:
     566        pass
     567
     568def ParseBoundaryStream(stream, max_header_size):
     569        """
     570        Parses one and exactly one stream that encapsulates a boundary.
     571        """
     572        # Stream at beginning of header, look for end of header
     573        # and parse it if found. The header must fit within one
     574        # chunk.
     575        chunk = stream.read(max_header_size)
     576        # 'find' returns the top of these four bytes, so we'll
     577        # need to munch them later to prevent them from polluting
     578        # the payload.
     579        header_end = chunk.find('\r\n\r\n')
     580
     581        def _parse_header(line):
     582            main_value_pair, params = parse_header(line)
     583            try:
     584                name, value = main_value_pair.split(':', 1)
     585            except:
     586                raise ValueError("Invalid header: %r" % line)
     587            return name, (value, params)
     588
     589        if header_end == -1:
     590            # we find no header, so we just mark this fact and pass on
     591            # the stream verbatim
     592            stream.unget(chunk)
     593            return (RAW, {}, stream)
     594
     595        header = chunk[:header_end]
     596
     597        # here we place any excess chunk back onto the stream, as
     598        # well as throwing away the CRLFCRLF bytes from above.
     599        stream.unget(chunk[header_end + 4:])
     600
     601        TYPE = RAW
     602        outdict = {}
     603
     604        # Eliminate blank lines
     605        for line in header.split('\r\n'):
     606            # This terminology ("main value" and "dictionary of
     607            # parameters") is from the Python docs.
     608            try:
     609                name, (value, params) = _parse_header(line)
     610            except:
     611                continue
     612
     613            if name == 'content-disposition':
     614                TYPE = FIELD
     615                if params.get('filename'):
     616                    TYPE = FILE
     617
     618            outdict[name] = value, params
     619
     620        if TYPE == RAW:
     621            stream.unget(chunk)
     622
     623        return (TYPE, outdict, stream)
     624
     625
     626class Parser(object):
     627    def __init__(self, stream, boundary):
     628        self._stream = stream
     629        self._separator = '--' + boundary
     630
     631    def __iter__(self):
     632
     633        boundarystream = InterBoundaryIter(self._stream,
     634                                           self._separator)
     635
     636        for sub_stream in boundarystream:
     637            # Iterate over each part
     638            yield ParseBoundaryStream(sub_stream, 1024)
     639
     640def parse_header(line):
     641    """ Parse the header into a key-value. """
     642    plist = _parse_header_params(';' + line)
     643    key = plist.pop(0).lower()
     644    pdict = {}
     645    for p in plist:
     646        i = p.find('=')
     647        if i >= 0:
     648            name = p[:i].strip().lower()
     649            value = p[i+1:].strip()
     650            if len(value) >= 2 and value[0] == value[-1] == '"':
     651                value = value[1:-1]
     652                value = value.replace('\\\\', '\\').replace('\\"', '"')
     653            pdict[name] = value
     654    return key, pdict
     655
     656def _parse_header_params(s):
     657    plist = []
     658    while s[:1] == ';':
     659        s = s[1:]
     660        end = s.find(';')
     661        while end > 0 and s.count('"', 0, end) % 2:
     662            end = s.find(';', end + 1)
     663        if end < 0:
     664            end = len(s)
     665        f = s[:end]
     666        plist.append(f.strip())
     667        s = s[end:]
     668    return plist
  • django/http/__init__.py

     
    99except ImportError:
    1010    from cgi import parse_qsl
    1111
    12 from django.utils.datastructures import MultiValueDict, FileDict
     12from django.utils.datastructures import MultiValueDict, ImmutableList
    1313from django.utils.encoding import smart_str, iri_to_uri, force_unicode
    14 
     14from django.http.multipartparser import MultiPartParser
    1515from utils import *
    1616
    1717RESERVED_CHARS="!*'();:@&=+$,/?%#[]"
     
    2525
    2626    # The encoding used in GET/POST dicts. None means use default setting.
    2727    _encoding = None
     28    _upload_handlers = ()
    2829
    2930    def __init__(self):
    3031        self.GET, self.POST, self.COOKIES, self.META, self.FILES = {}, {}, {}, {}, {}
     
    102103
    103104    encoding = property(_get_encoding, _set_encoding)
    104105
    105 def parse_file_upload(header_dict, post_data):
    106     """Returns a tuple of (POST QueryDict, FILES MultiValueDict)."""
    107     import email, email.Message
    108     from cgi import parse_header
    109     raw_message = '\r\n'.join(['%s:%s' % pair for pair in header_dict.items()])
    110     raw_message += '\r\n\r\n' + post_data
    111     msg = email.message_from_string(raw_message)
    112     POST = QueryDict('', mutable=True)
    113     FILES = MultiValueDict()
    114     for submessage in msg.get_payload():
    115         if submessage and isinstance(submessage, email.Message.Message):
    116             name_dict = parse_header(submessage['Content-Disposition'])[1]
    117             # name_dict is something like {'name': 'file', 'filename': 'test.txt'} for file uploads
    118             # or {'name': 'blah'} for POST fields
    119             # We assume all uploaded files have a 'filename' set.
    120             if 'filename' in name_dict:
    121                 assert type([]) != type(submessage.get_payload()), "Nested MIME messages are not supported"
    122                 if not name_dict['filename'].strip():
    123                     continue
    124                 # IE submits the full path, so trim everything but the basename.
    125                 # (We can't use os.path.basename because that uses the server's
    126                 # directory separator, which may not be the same as the
    127                 # client's one.)
    128                 filename = name_dict['filename'][name_dict['filename'].rfind("\\")+1:]
    129                 FILES.appendlist(name_dict['name'], FileDict({
    130                     'filename': filename,
    131                     'content-type': 'Content-Type' in submessage and submessage['Content-Type'] or None,
    132                     'content': submessage.get_payload(),
    133                 }))
    134             else:
    135                 POST.appendlist(name_dict['name'], submessage.get_payload())
    136     return POST, FILES
     106    def _initialize_handlers(self):
     107        from django.conf import settings
     108        from django.core.files.fileuploadhandler import load_handler
     109        handlers = []
     110        # We go through each handler in the settings variable
     111        # and instantiate the handler by calling HandlerClass(request).
     112        for handler in settings.FILE_UPLOAD_HANDLERS:
     113            handlers.append(load_handler(handler, self))
     114        self._upload_handlers = handlers
    137115
     116    def _set_upload_handlers(self, upload_handlers):
     117        """
     118        Set the upload handler to the new handler given in the parameter.
     119        """
     120        if hasattr(self, '_files'):
     121            raise AttributeError("You cannot set the upload handlers after the upload has been processed.")
     122        self._upload_handlers = upload_handlers
    138123
     124    def _get_upload_handlers(self):
     125        if not self._upload_handlers:
     126            # If thre are no upload handlers defined, initialize them from settings.
     127            self._initialize_handlers()
     128        return self._upload_handlers
     129
     130    upload_handlers = property(_get_upload_handlers, _set_upload_handlers)
     131
     132    def parse_file_upload(self, META, post_data):
     133        """Returns a tuple of (POST QueryDict, FILES MultiValueDict)."""
     134        self.upload_handlers = ImmutableList(self.upload_handlers,
     135                                             warning="You cannot alter the upload handlers after the upload has been processed.")
     136        parser = MultiPartParser(META, post_data, self.upload_handlers,
     137                                 self.encoding)
     138        return parser.parse()
     139
     140
    139141class QueryDict(MultiValueDict):
    140142    """
    141143    A specialized MultiValueDict that takes a query string when initialized.
  • django/test/client.py

     
    1818BOUNDARY = 'BoUnDaRyStRiNg'
    1919MULTIPART_CONTENT = 'multipart/form-data; boundary=%s' % BOUNDARY
    2020
     21
     22class FakePayload(object):
     23    """
     24    A wrapper around StringIO that restricts what can be read,
     25    since data from the network can't be seeked and cannot
     26    be read outside of its content length (or else we hang).
     27    """
     28    def __init__(self, content):
     29        self.__content = StringIO(content)
     30        self.__len = len(content)
     31
     32    def read(self, num_bytes=None):
     33        if num_bytes is None:
     34            num_bytes = self.__len or 1
     35        assert self.__len >= num_bytes, "Cannot read more than the available bytes from the HTTP incoming data."
     36        content = self.__content.read(num_bytes)
     37        self.__len -= num_bytes
     38        return content
     39
     40
    2141class ClientHandler(BaseHandler):
    2242    """
    2343    A HTTP Handler that can be used for testing purposes.
     
    230250            'CONTENT_TYPE':   content_type,
    231251            'PATH_INFO':      urllib.unquote(path),
    232252            'REQUEST_METHOD': 'POST',
    233             'wsgi.input':     StringIO(post_data),
     253            'wsgi.input':     FakePayload(post_data),
    234254        }
    235255        r.update(extra)
    236256
  • django/conf/global_settings.py

     
    114114SEND_BROKEN_LINK_EMAILS = False
    115115
    116116# Database connection info.
    117 DATABASE_ENGINE = ''           # 'postgresql_psycopg2', 'postgresql', 'mysql', 'sqlite3' or 'oracle'.
     117DATABASE_ENGINE = ''           # 'postgresql_psycopg2', 'postgresql', 'mysql', 'sqlite3' or 'ado_mssql'.
    118118DATABASE_NAME = ''             # Or path to database file if using sqlite3.
    119119DATABASE_USER = ''             # Not used with sqlite3.
    120120DATABASE_PASSWORD = ''         # Not used with sqlite3.
     
    224224# Example: "http://media.lawrence.com"
    225225MEDIA_URL = ''
    226226
     227# A tuple that enumerates the upload handlers
     228# in order.
     229FILE_UPLOAD_HANDLERS = (
     230    'django.core.files.fileuploadhandler.MemoryFileUploadHandler',
     231    'django.core.files.fileuploadhandler.TemporaryFileUploadHandler',
     232)
     233
     234# Number of bytes the length of the request can be before it is
     235# streamed to the file system instead of parsed entirely in memory.
     236FILE_UPLOAD_MAX_MEMORY_SIZE = 2621440
     237
     238# Directory to upload streamed files temporarily.
     239# A value of `None` means that it will use the default temporary
     240# directory for the server's operating system.
     241FILE_UPLOAD_TEMP_DIR = None
     242
    227243# Default formatting for date objects. See all available format strings here:
    228244# http://www.djangoproject.com/documentation/templates/#now
    229245DATE_FORMAT = 'N j, Y'
  • django/db/models/base.py

     
    1313from django.utils.datastructures import SortedDict
    1414from django.utils.functional import curry
    1515from django.utils.encoding import smart_str, force_unicode, smart_unicode
     16from django.core.files.filemove import file_move_safe
    1617from django.conf import settings
    1718from itertools import izip
     19import warnings
    1820import types
    1921import sys
    2022import os
     
    384386    def _get_FIELD_size(self, field):
    385387        return os.path.getsize(self._get_FIELD_filename(field))
    386388
    387     def _save_FIELD_file(self, field, filename, raw_contents, save=True):
     389    def _save_FIELD_file(self, field, filename, raw_field, save=True):
    388390        directory = field.get_directory_name()
    389391        try: # Create the date-based directory if it doesn't exist.
    390392            os.makedirs(os.path.join(settings.MEDIA_ROOT, directory))
    391393        except OSError: # Directory probably already exists.
    392394            pass
     395
     396        # Put the deprecation warning first since there are multiple
     397        # locations where we use the new and old interface.
     398        if isinstance(raw_field, dict):
     399            warnings.warn("The dictionary usage for files is deprecated. Use the new object interface instead.", DeprecationWarning)
     400
     401        if filename is None:
     402            try:
     403                filename = raw_field.file_name
     404            except AttributeError:
     405                filename = raw_field['filename']
     406
    393407        filename = field.get_filename(filename)
    394408
    395409        # If the filename already exists, keep adding an underscore to the name of
     
    406420        setattr(self, field.attname, filename)
    407421
    408422        full_filename = self._get_FIELD_filename(field)
    409         fp = open(full_filename, 'wb')
    410         fp.write(raw_contents)
    411         fp.close()
     423        if hasattr(raw_field, 'temporary_file_path'):
     424            # This file has a file path that we can move.
     425            raw_field.close()
     426            file_move_safe(raw_field.temporary_file_path(), full_filename)
     427        else:
     428            from django.core.files import filelocks
     429            fp = open(full_filename, 'wb')
     430            # exclusive lock
     431            filelocks.lock(fp, filelocks.LOCK_EX)
     432            if hasattr(raw_field, 'chunk'):
     433                # This is a normal uploadedfile that we can stream.
     434                for chunk in raw_field.chunk(65535):
     435                    fp.write(chunk)
     436            else:
     437                # This is an old dictionary, use the old interface.
     438                fp.write(raw_field['content'])
     439            filelocks.unlock(fp)
     440            fp.close()
    412441
     442
    413443        # Save the width and/or height, if applicable.
    414444        if isinstance(field, ImageField) and (field.width_field or field.height_field):
    415445            from django.utils.images import get_image_dimensions
  • django/db/models/fields/__init__.py

     
    785785        setattr(cls, 'get_%s_filename' % self.name, curry(cls._get_FIELD_filename, field=self))
    786786        setattr(cls, 'get_%s_url' % self.name, curry(cls._get_FIELD_url, field=self))
    787787        setattr(cls, 'get_%s_size' % self.name, curry(cls._get_FIELD_size, field=self))
    788         setattr(cls, 'save_%s_file' % self.name, lambda instance, filename, raw_contents, save=True: instance._save_FIELD_file(self, filename, raw_contents, save))
     788        setattr(cls, 'save_%s_file' % self.name, lambda instance, filename, raw_field, save=True: instance._save_FIELD_file(self, filename, raw_field, save))
     789        setattr(cls, 'move_%s_file' % self.name, lambda instance, raw_field, save=True: instance._save_FIELD_file(self, None, raw_field, save))
    789790        dispatcher.connect(self.delete_file, signal=signals.post_delete, sender=cls)
    790791
    791792    def delete_file(self, instance):
     
    808809        if new_data.get(upload_field_name, False):
    809810            func = getattr(new_object, 'save_%s_file' % self.name)
    810811            if rel:
    811                 func(new_data[upload_field_name][0]["filename"], new_data[upload_field_name][0]["content"], save)
     812                file = new_data[upload_field_name][0]
    812813            else:
    813                 func(new_data[upload_field_name]["filename"], new_data[upload_field_name]["content"], save)
     814                file = new_data[upload_field_name]
    814815
     816            try:
     817                file_name = file.file_name
     818            except AttributeError:
     819                file_name = file['filename']
     820            func(file_name, file, save)
     821
    815822    def get_directory_name(self):
    816823        return os.path.normpath(force_unicode(datetime.datetime.now().strftime(smart_str(self.upload_to))))
    817824
     
    823830    def save_form_data(self, instance, data):
    824831        from django.newforms.fields import UploadedFile
    825832        if data and isinstance(data, UploadedFile):
    826             getattr(instance, "save_%s_file" % self.name)(data.filename, data.content, save=False)
     833            getattr(instance, "save_%s_file" % self.name)(data.filename, data.data, save=False)
    827834
    828835    def formfield(self, **kwargs):
    829836        defaults = {'form_class': forms.FileField}
  • django/oldforms/__init__.py

     
    680680        self.field_name, self.is_required = field_name, is_required
    681681        self.validator_list = [self.isNonEmptyFile] + validator_list
    682682
    683     def isNonEmptyFile(self, field_data, all_data):
     683    def isNonEmptyFile(self, new_data, all_data):
     684        if hasattr(new_data, 'upload_errors'):
     685            upload_errors = new_data.upload_errors()
     686            if upload_errors:
     687                raise validators.CriticalValidationError, upload_errors
    684688        try:
    685             content = field_data['content']
    686         except TypeError:
    687             raise validators.CriticalValidationError, ugettext("No file was submitted. Check the encoding type on the form.")
    688         if not content:
     689            file_size = new_data.file_size
     690        except AttributeError:
     691            file_size = len(new_data['content'])
     692        if not file_size:
    689693            raise validators.CriticalValidationError, ugettext("The submitted file is empty.")
    690694
    691695    def render(self, data):
    692696        return mark_safe(u'<input type="file" id="%s" class="v%s" name="%s" />' % \
    693697            (self.get_id(), self.__class__.__name__, self.field_name))
    694698
     699    def prepare(self, new_data):
     700        if hasattr(new_data, 'upload_errors'):
     701            upload_errors = new_data.upload_errors()
     702            new_data[self.field_name] = { '_file_upload_error': upload_errors }
     703
    695704    def html2python(data):
    696705        if data is None:
    697706            raise EmptyValue
  • django/core/handlers/wsgi.py

     
    112112        # Populates self._post and self._files
    113113        if self.method == 'POST':
    114114            if self.environ.get('CONTENT_TYPE', '').startswith('multipart'):
    115                 header_dict = dict([(k, v) for k, v in self.environ.items() if k.startswith('HTTP_')])
    116                 header_dict['Content-Type'] = self.environ.get('CONTENT_TYPE', '')
    117                 self._post, self._files = http.parse_file_upload(header_dict, self.raw_post_data)
     115                self._raw_post_data = ''
     116                self._post, self._files = self.parse_file_upload(self.META, self.environ['wsgi.input'])
    118117            else:
    119118                self._post, self._files = http.QueryDict(self.raw_post_data, encoding=self._encoding), datastructures.MultiValueDict()
    120119        else:
  • django/core/handlers/modpython.py

     
    5353    def _load_post_and_files(self):
    5454        "Populates self._post and self._files"
    5555        if 'content-type' in self._req.headers_in and self._req.headers_in['content-type'].startswith('multipart'):
    56             self._post, self._files = http.parse_file_upload(self._req.headers_in, self.raw_post_data)
     56            self._raw_post_data = ''
     57            self._post, self._files = self.parse_file_upload(self.META, self._req)
    5758        else:
    5859            self._post, self._files = http.QueryDict(self.raw_post_data, encoding=self._encoding), datastructures.MultiValueDict()
    5960
  • django/core/files/filelocks.py

     
     1"""
     2Locking portability based partially on example by
     3Jonathan Feignberg <jdf@pobox.com> in python cookbook.
     4
     5Example Usage::
     6
     7    from django.core.files import filelocks
     8
     9    f = open('./file', 'wb')
     10
     11    filelocks.lock(f, filelocks.LOCK_EX)
     12    f.write('Django')
     13    f.close()
     14"""
     15
     16__all__ = ('LOCK_EX','LOCK_SH','LOCK_NB','lock','unlock')
     17
     18system_type = None
     19
     20try:
     21    import win32con
     22    import win32file
     23    import pywintypes
     24    LOCK_EX = win32con.LOCKFILE_EXCLUSIVE_LOCK
     25    LOCK_SH = 0
     26    LOCK_NB = win32con.LOCKFILE_FAIL_IMMEDIATELY
     27    __overlapped = pywintypes.OVERLAPPED()
     28    system_type = 'nt'
     29except (ImportError, AttributeError):
     30    pass
     31
     32try:
     33    import fcntl
     34    LOCK_EX = fcntl.LOCK_EX
     35    LOCK_SH = fcntl.LOCK_SH
     36    LOCK_NB = fcntl.LOCK_NB
     37    system_type = 'posix'
     38except (ImportError, AttributeError):
     39    pass
     40
     41if system_type == 'nt':
     42    def lock(file, flags):
     43        hfile = win32file._get_osfhandle(file.fileno())
     44        win32file.LockFileEx(hfile, flags, 0, -0x10000, __overlapped)
     45
     46    def unlock(file):
     47        hfile = win32file._get_osfhandle(file.fileno())
     48        win32file.UnlockFileEx(hfile, 0, -0x10000, __overlapped)
     49elif system_type == 'posix':
     50    def lock(file, flags):
     51        fcntl.flock(file.fileno(), flags)
     52
     53    def unlock(file):
     54        fcntl.flock(file.fileno(), fcntl.LOCK_UN)
     55else:
     56    # File locking is not supported.
     57    LOCK_EX = LOCK_SH = LOCK_NB = None
     58       
     59    # Dummy functions that don't do anything.
     60    def lock(file, flags):
     61        pass
     62
     63    def unlock(file):
     64        pass
  • django/core/files/uploadedfile.py

     
     1"""
     2The uploaded file objects for Django.
     3This contains the base UploadedFile and the TemporaryUploadedFile
     4derived class.
     5"""
     6
     7__all__ = ('UploadedFile', 'TemporaryUploadedFile', 'InMemoryUploadedFile')
     8
     9class UploadedFile(object):
     10    """
     11    The UploadedFile object behaves somewhat like a file
     12    object and represents some data that the user submitted
     13    and is stored in some form.
     14    """
     15    DEFAULT_CHUNK_SIZE = 64 * 2**10
     16
     17    def __init__(self):
     18        self.file_size = None
     19        self.file_name = None
     20        self.content_type = None
     21        self.charset = None
     22        pass
     23
     24    def file_size(self):
     25        return self.file_size
     26
     27    def chunk(self, chunk_size=None):
     28        """
     29        Read the file to generate chunks of chunk_size bytes.
     30        """
     31        if not chunk_size:
     32            chunk_size = UploadedFile.DEFAULT_CHUNK_SIZE
     33
     34        if hasattr(self, 'seek'):
     35            self.seek(0)
     36        # Assume the pointer is at zero...
     37        counter = self.file_size()
     38
     39        while counter > 0:
     40            yield self.read(chunk_size)
     41            counter -= chunk_size
     42
     43
     44    def multiple_chunks(self, chunk_size=None):
     45        """
     46        Return True if you can expect multiple chunks, False otherwise.
     47        Note: If a particular file representation is in memory, then
     48              override this to return False.
     49        """
     50        if not chunk_size:
     51            chunk_size = UploadedFile.DEFAULT_CHUNK_SIZE
     52        return self.file_size() < chunk_size
     53       
     54
     55    def read(self, num_bytes=None):
     56        """
     57        Read from the file in whatever representation it has.
     58        """
     59        raise NotImplementedError()
     60
     61    def open(self):
     62        """
     63        Open the file, if one needs to.
     64        """
     65        pass
     66
     67
     68    def close(self):
     69        """
     70        Close the file, if one needs to.
     71        """
     72        pass
     73
     74    def __getitem__(self, key):
     75        """
     76        This maintains backwards compatibility.
     77        """
     78        import warnings
     79        warnings.warn("The dictionary access of uploaded file objects is deprecated. Use the new object interface instead.", DeprecationWarning)
     80        # Dictionary to translate labels
     81        # for backwards compatbility.
     82        # Should be removed at some point.
     83        backwards_translate = {
     84            'filename': 'file_name',
     85            'content-type': 'content_type',
     86            }
     87
     88        if key == 'content':
     89            return self.read()
     90        else:
     91            return getattr(self, backwards_translate.get(key, key))
     92
     93    def __repr__(self):
     94        """
     95        This representation could be anything and can be overridden.
     96        This is mostly done to make it look somewhat useful.
     97        """
     98        _dict = {
     99            'file_name': self.file_name,
     100            'content_type': self.content_type,
     101            'content': '<omitted>',
     102            }
     103        return repr(_dict)
     104
     105
     106class TemporaryUploadedFile(UploadedFile):
     107    """
     108    Upload a file to a temporary file.
     109    """
     110
     111    def __init__(self, file, file_name, content_type, file_size, charset):
     112        self.file = file
     113        self.file_name = file_name
     114        self.path = file.name
     115        self.content_type = content_type
     116        self.file_size = file_size
     117        self.charset = charset
     118        self.file.seek(0)
     119
     120    def temporary_file_path(self):
     121        """
     122        Return the full path of this file.
     123        """
     124        return self.path
     125
     126    def read(self, *args, **kwargs):
     127        return self.file.read(*args, **kwargs)
     128
     129    def open(self):
     130        """
     131        Assume the person meant to seek.
     132        """
     133        self.seek(0)
     134
     135    def seek(self, *args, **kwargs):
     136        self.file.seek(*args, **kwargs)
     137
     138
     139class InMemoryUploadedFile(UploadedFile):
     140    """
     141    Upload a file into memory.
     142    """
     143    def __init__(self, file, field_name, file_name, content_type, charset):
     144        self.file = file
     145        self.field_name = field_name
     146        self.file_name = file_name
     147        self.content_type = content_type
     148        self.charset = charset
     149        self.file.seek(0)
     150
     151    def seek(self, *args, **kwargs):
     152        self.file.seek(*args, **kwargs)
     153
     154    def open(self):
     155        self.seek(0)
     156
     157    def read(self, *args, **kwargs):
     158        return self.file.read(*args, **kwargs)
     159
     160    def chunk(self, chunk_size=None):
     161        """
     162        Return the entirety of the data regardless.
     163        """
     164        self.file.seek(0)
     165        return self.read()
     166
     167    def multiple_chunks(self, chunk_size=None):
     168        """
     169        Since it's in memory, we'll never have multiple chunks.
     170        """
     171        return False
     172
     173
     174class SimpleUploadedFile(InMemoryUploadedFile):
     175    """
     176    A simple representation of a file, which
     177    just has content, size, and a name.
     178    """
     179    def __init__(self, name, content, content_type='text/plain'):
     180        try:
     181            from cStringIO import StringIO
     182        except ImportError:
     183            from StringIO import StringIO
     184        self.file = StringIO(content or '')
     185        self.file_name = name
     186        self.field_name = None
     187        self.file_size = len(content or '')
     188        self.content_type = content_type
     189        self.charset = None
     190        self.file.seek(0)
     191
  • django/core/files/__init__.py

     
     1
  • django/core/files/fileuploadhandler.py

     
     1""" A fileuploadhandler base and default subclass for handling file uploads.
     2"""
     3import os
     4try:
     5    from cStringIO import StringIO
     6except ImportError:
     7    from StringIO import StringIO
     8
     9from django.utils.encoding import force_unicode
     10from django.utils.datastructures import MultiValueDict
     11from django.core.exceptions import ImproperlyConfigured
     12
     13from django.core.files.uploadedfile import TemporaryUploadedFile, InMemoryUploadedFile
     14
     15__all__ = ('UploadFileException','StopUpload', 'SkipFile',
     16           'FileUploadHandler', 'TemporaryFileUploadHandler',
     17           'MemoryFileUploadHandler', 'load_handler')
     18
     19
     20class UploadFileException(Exception):
     21    """ Any error having to do with Uploading Files. """
     22    pass
     23
     24class StopUpload(UploadFileException):
     25    """ This exception is raised when an upload must abort. """
     26    pass
     27
     28class SkipFile(UploadFileException):
     29    """ This exception is raised when a file needs to be skipped. """
     30    pass
     31
     32def load_handler(path, *args, **kwargs):
     33    """
     34    Given a path to a handler return the instantiation of that handler.
     35    E.g.::
     36        load_handler('django.core.files.fileuploadhandler.TemporaryFileUploadHandler', request)
     37    will return a TemporaryFileUploadHandler object.
     38    """
     39    i = path.rfind('.')
     40    module, attr = path[:i], path[i+1:]
     41    try:
     42        mod = __import__(module, {}, {}, [attr])
     43    except ImportError, e:
     44        raise ImproperlyConfigured, 'Error importing upload handler module %s: "%s"' % (module, e)
     45    except ValueError, e:
     46        raise ImproperlyConfigured, 'Error importing upload handler module Is FILE_UPLOAD_HANDLERS a correctly defined list or tuple?'
     47    try:
     48        cls = getattr(mod, attr)
     49    except AttributeError:
     50        raise ImproperlyConfigured, 'Module "%s" does not define a "%s" upload handler backend' % (module, attr)
     51    return cls(*args, **kwargs)
     52
     53
     54class FileUploadHandler(object):
     55    """ FileUploadHandler will take data and handle file uploads
     56    in a streamed fashion.
     57    """
     58    chunk_size = 64 * 2 ** 10 #: The default chunk size is 64 KB.
     59
     60    def __init__(self, request=None):
     61        " Initialize some local variables. "
     62        self.file_name = None
     63        self.content_type = None
     64        self.content_length = None
     65        self.charset = None
     66        self.request = request
     67
     68    def handle_raw_input(self, input_data, META, content_length, boundary, encoding=None):
     69        """
     70        Handle the raw input from the client.
     71        Parameters:
     72          *input_data* -- An object that supports reading via .read().
     73          *content_length* -- The (integer) value of the Content-Length header from the client.
     74          *boundary* -- The boundary from the Content-Type header. Be sure to prepend two '--'.
     75        """
     76        pass
     77
     78    def new_file(self, field_name, file_name, content_type, content_length, charset=None):
     79        """
     80        Signal that a new file has been started.
     81       
     82        Warning: Do not trust content_length, if you get it at all.
     83        """
     84        self.field_name = field_name
     85        self.file_name = file_name
     86        self.content_type = content_type
     87        self.content_length = content_length
     88        self.charset = charset
     89
     90    def receive_data_chunk(self, raw_data, start, stop):
     91        """
     92        Receive data from the streamed upload parser.
     93        Start and stop are the positions in the file.
     94        This equality should always be true::
     95            len(raw_data) = stop - start
     96        """
     97        raise NotImplementedError()
     98
     99    def file_complete(self, file_size):
     100        """
     101        Signal that a file has completed.
     102        File size corresponds to the actual size accumulated
     103        by all the chunks.
     104
     105        This should return a valid UploadedFile object.
     106        """
     107        raise NotImplementedError()
     108
     109    def upload_complete(self):
     110        """
     111        Signal that the upload is complete.
     112        Do any cleanup that is necessary for this handler.
     113        """
     114        pass
     115
     116
     117
     118class TemporaryFileUploadHandler(FileUploadHandler):
     119    """
     120    Upload the streaming data into a temporary file.
     121    """
     122    def __init__(self, *args, **kwargs):
     123        """ Import settings for later. """
     124        super(TemporaryFileUploadHandler, self).__init__(*args, **kwargs)
     125        global settings
     126        from django.conf import settings
     127
     128    def new_file(self, file_name, *args, **kwargs):
     129        """
     130        Create the file object to append to as data is coming in.
     131        """
     132        super(TemporaryFileUploadHandler, self).new_file(file_name, *args, **kwargs)
     133        self.file = TemporaryFile(settings.FILE_UPLOAD_TEMP_DIR)
     134        self.write = self.file.write
     135
     136    def receive_data_chunk(self, raw_data, start, stop):
     137        """
     138        Once we get the data, we will save it to our file.
     139        """
     140        self.write(raw_data)
     141
     142    def file_complete(self, file_size):
     143        """
     144        Signal that a file has completed.
     145        File size corresponds to the actual size accumulated
     146        by all the chunks.
     147
     148        This should return a valid UploadedFile object.
     149        """
     150        self.file.seek(0)
     151        return TemporaryUploadedFile(self.file, self.file_name,
     152                                     self.content_type, file_size,
     153                                     self.charset)
     154
     155
     156class MemoryFileUploadHandler(FileUploadHandler):
     157    """
     158    The MemoryFileUploadHandler will place the data directly into memory.
     159    """
     160
     161    def handle_raw_input(self, input_data, META, content_length, boundary, encoding=None):
     162        """
     163        Use the content_length to signal whether or not this handler should be in use.
     164        """
     165        global settings
     166        from django.conf import settings
     167
     168        # If the the post is too large, we cannot use the Memory handler.
     169        if content_length > settings.FILE_UPLOAD_MAX_MEMORY_SIZE:
     170            self.activated = False
     171        else:
     172            self.activated = True
     173
     174    def new_file(self, *args, **kwargs):
     175        super(MemoryFileUploadHandler, self).new_file(*args, **kwargs)
     176        if self.activated:
     177            self.file = StringIO()
     178            return "Stop"
     179
     180    def receive_data_chunk(self, raw_data, start, stop):
     181        """
     182        Add the data to the StringIO file.
     183        """
     184        if self.activated:
     185            self.file.write(raw_data)
     186        else:
     187            return raw_data
     188
     189    def file_complete(self, file_size):
     190        """
     191        Return a file object if we're activated.
     192        """
     193        if not self.activated:
     194            return
     195
     196        return InMemoryUploadedFile(self.file, self.field_name, self.file_name,
     197                                    self.content_type, self.charset)
     198
     199
     200class TemporaryFile(object):
     201    """
     202    A temporary file that tries to delete itself when garbage collected.
     203    """
     204    def __init__(self, dir):
     205        import tempfile
     206        if not dir:
     207            dir = tempfile.gettempdir()
     208        try:
     209            (fd, name) = tempfile.mkstemp(suffix='.upload', dir=dir)
     210            self.file = os.fdopen(fd, 'w+b')
     211        except (OSError, IOError):
     212            raise OSError, "Could not create temporary file for uploading, have you set settings.FILE_UPLOAD_TEMP_DIR correctly?"
     213        self.name = name
     214
     215    def __getattr__(self, name):
     216        a = getattr(self.__dict__['file'], name)
     217        if type(a) != type(0):
     218            setattr(self, name, a)
     219        return a
     220
     221    def __del__(self):
     222        try:
     223            os.unlink(self.name)
     224        except OSError:
     225            pass
  • django/core/files/filemove.py

     
     1import os
     2
     3__all__ = ('file_move_safe',)
     4
     5try:
     6    import shutil
     7    file_move = shutil.move
     8except ImportError:
     9    file_move = os.rename
     10
     11def file_move_safe(old_file_name, new_file_name, chunk_size = 1024*64, allow_overwrite=False):
     12    """
     13    Moves a file from one location to another in the safest way possible.
     14   
     15    First, it tries using shutils.move, which is OS-dependent but doesn't
     16    break with change of filesystems. Then it tries os.rename, which will
     17    break if it encounters a change in filesystems. Lastly, it streams
     18    it manually from one file to another in python.
     19
     20    Without ``allow_overwrite``, if the destination file exists, the
     21    file will raise an IOError.
     22    """
     23
     24    from django.core.files import filelocks
     25
     26    if old_file_name == new_file_name:
     27        # No file moving takes place.
     28        return
     29
     30    if not allow_overwrite and os.path.exists(new_file_name):
     31        raise IOError, "Django does not allow overwriting files."
     32
     33    try:
     34        file_move(old_file_name, new_file_name)
     35        return
     36    except OSError: # moving to another filesystem
     37        pass
     38
     39    new_file = open(new_file_name, 'wb')
     40    # exclusive lock
     41    filelocks.lock(new_file, filelocks.LOCK_EX)
     42    old_file = open(old_file_name, 'rb')
     43    current_chunk = None
     44
     45    while current_chunk != '':
     46        current_chunk = old_file.read(chunk_size)
     47        new_file.write(current_chunk)
     48
     49    new_file.close()
     50    old_file.close()
     51
     52    os.remove(old_file_name)
  • django/utils/datastructures.py

     
    332332            except TypeError: # Special-case if current isn't a dict.
    333333                current = {bits[-1]: v}
    334334
    335 class FileDict(dict):
     335class ImmutableList(tuple):
    336336    """
    337     A dictionary used to hold uploaded file contents. The only special feature
    338     here is that repr() of this object won't dump the entire contents of the
    339     file to the output. A handy safeguard for a large file upload.
     337    A tuple-like object that raises useful
     338    errors when it is asked to mutate.
     339    Example::
     340        a = ImmutableList(range(5), warning=AttributeError("You cannot mutate this."))
     341        a[3] = '4'
     342        (Raises the AttributeError)
    340343    """
    341     def __repr__(self):
    342         if 'content' in self:
    343             d = dict(self, content='<omitted>')
    344             return dict.__repr__(d)
    345         return dict.__repr__(self)
     344
     345    def __new__(cls, *args, **kwargs):
     346        if 'warning' in kwargs:
     347            warning = kwargs['warning']
     348            del kwargs['warning']
     349        else:
     350            warning = 'ImmutableList object is immutable.'
     351        self = tuple.__new__(cls, *args, **kwargs)
     352        self.warning = warning
     353        return self
     354
     355    def complain(self, *wargs, **kwargs):
     356        if isinstance(self.warning, Exception):
     357            raise self.warning
     358        else:
     359            raise AttributeError, self.warning
     360
     361    # All list mutation functions become complain.
     362    __delitem__ = __delslice__ = __iadd__ = __imul__ = complain
     363    __setitem__ = __setslice__ = complain
     364    append = extend = insert = pop = remove = complain
     365    sort = reverse = complain
  • django/utils/text.py

     
    33from django.utils.encoding import force_unicode
    44from django.utils.functional import allow_lazy
    55from django.utils.translation import ugettext_lazy
     6from htmlentitydefs import name2codepoint
    67
    78# Capitalizes the first letter of a string.
    89capfirst = lambda x: x and force_unicode(x)[0].upper() + force_unicode(x)[1:]
     
    218219            yield bit
    219220smart_split = allow_lazy(smart_split, unicode)
    220221
     222def _replace_entity(match):
     223     text = match.group(1)
     224     if text[0] == u'#':
     225         text = text[1:]
     226         try:
     227             if text[0] in u'xX':
     228                 c = int(text[1:], 16)
     229             else:
     230                 c = int(text)
     231             return unichr(c)
     232         except ValueError:
     233             return match.group(0)
     234     else:
     235         try:
     236             return unichr(name2codepoint[text])
     237         except (ValueError, KeyError):
     238             return match.group(0)
     239
     240_entity_re = re.compile(r"&(#?[xX]?(?:[0-9a-fA-F]+|\w{1,8}));")
     241
     242def unescape_entities(text):
     243     return _entity_re.sub(_replace_entity, text)
     244unescape_entities = allow_lazy(unescape_entities, unicode)
  • django/newforms/fields.py

     
    44
    55import copy
    66import datetime
     7import warnings
    78import os
    89import re
    910import time
     
    416417
    417418class UploadedFile(StrAndUnicode):
    418419    "A wrapper for files uploaded in a FileField"
    419     def __init__(self, filename, content):
     420    def __init__(self, filename, data):
    420421        self.filename = filename
    421         self.content = content
     422        self.data = data
    422423
    423424    def __unicode__(self):
    424425        """
     
    444445            return None
    445446        elif not data and initial:
    446447            return initial
     448
     449        if isinstance(data, dict):
     450            # We warn once, then support both ways below.
     451            warnings.warn("The dictionary usage for files is deprecated. Use the new object interface instead.", DeprecationWarning)
     452
    447453        try:
    448             f = UploadedFile(data['filename'], data['content'])
    449         except TypeError:
     454            file_name = data.file_name
     455            file_size = data.file_size
     456        except AttributeError:
     457            try:
     458                file_name = data.get('filename')
     459                file_size = bool(data['content'])
     460            except (AttributeError, KeyError):
     461                raise ValidationError(self.error_messages['invalid'])
     462
     463        if not file_name:
    450464            raise ValidationError(self.error_messages['invalid'])
    451         except KeyError:
    452             raise ValidationError(self.error_messages['missing'])
    453         if not f.content:
     465        if not file_size:
    454466            raise ValidationError(self.error_messages['empty'])
    455         return f
    456467
     468        return UploadedFile(file_name, data)
     469
    457470class ImageField(FileField):
    458471    default_error_messages = {
    459472        'invalid_image': _(u"Upload a valid image. The file you uploaded was either not an image or a corrupted image."),
     
    470483        elif not data and initial:
    471484            return initial
    472485        from PIL import Image
    473         from cStringIO import StringIO
     486
     487        # We need to get the file, it either has a path
     488        # or we have to read it all into memory...
     489        if hasattr(data, 'temporary_file_path'):
     490            file = data.temporary_file_path()
     491        else:
     492            try:
     493                from cStringIO import StringIO
     494            except ImportError:
     495                from StringIO import StringIO
     496            if hasattr(data, 'read'):
     497                file = StringIO(data.read())
     498            else:
     499                file = StringIO(data['content'])
    474500        try:
    475501            # load() is the only method that can spot a truncated JPEG,
    476502            #  but it cannot be called sanely after verify()
    477             trial_image = Image.open(StringIO(f.content))
     503            trial_image = Image.open(file)
    478504            trial_image.load()
    479505            # verify() is the only method that can spot a corrupt PNG,
    480506            #  but it must be called immediately after the constructor
    481             trial_image = Image.open(StringIO(f.content))
     507            trial_image = Image.open(file)
    482508            trial_image.verify()
    483509        except Exception: # Python Imaging Library doesn't recognize it as an image
    484510            raise ValidationError(self.error_messages['invalid_image'])
  • tests/modeltests/model_forms/models.py

     
    7575__test__ = {'API_TESTS': """
    7676>>> from django import newforms as forms
    7777>>> from django.newforms.models import ModelForm
     78>>> from django.core.files.uploadedfile import SimpleUploadedFile
     79>>> from warnings import filterwarnings
     80>>> filterwarnings("ignore")
    7881
    7982The bare bones, absolutely nothing custom, basic case.
    8083
     
    792795
    793796# Upload a file and ensure it all works as expected.
    794797
     798>>> f = TextFileForm(data={'description': u'Assistance'}, files={'file': SimpleUploadedFile('test1.txt', 'hello world')})
     799>>> f.is_valid()
     800True
     801>>> type(f.cleaned_data['file'])
     802<class 'django.newforms.fields.UploadedFile'>
     803>>> instance = f.save()
     804>>> instance.file
     805u'...test1.txt'
     806
     807>>> os.unlink(instance.get_file_filename())
     808
    795809>>> f = TextFileForm(data={'description': u'Assistance'}, files={'file': {'filename': 'test1.txt', 'content': 'hello world'}})
    796810>>> f.is_valid()
    797811True
     
    814828u'...test1.txt'
    815829
    816830# Delete the current file since this is not done by Django.
    817 
    818831>>> os.unlink(instance.get_file_filename())
    819832
    820833# Override the file by uploading a new one.
    821834
    822 >>> f = TextFileForm(data={'description': u'Assistance'}, files={'file': {'filename': 'test2.txt', 'content': 'hello world'}}, instance=instance)
     835>>> f = TextFileForm(data={'description': u'Assistance'}, files={'file': SimpleUploadedFile('test2.txt', 'hello world')}, instance=instance)
    823836>>> f.is_valid()
    824837True
    825838>>> instance = f.save()
    826839>>> instance.file
    827840u'...test2.txt'
    828841
     842# Delete the current file since this is not done by Django.
     843>>> os.unlink(instance.get_file_filename())
     844
     845>>> f = TextFileForm(data={'description': u'Assistance'}, files={'file': {'filename': 'test2.txt', 'content': 'hello world'}})
     846>>> f.is_valid()
     847True
     848>>> instance = f.save()
     849>>> instance.file
     850u'...test2.txt'
     851
     852# Delete the current file since this is not done by Django.
     853>>> os.unlink(instance.get_file_filename())
     854
    829855>>> instance.delete()
    830856
    831857# Test the non-required FileField
     
    838864>>> instance.file
    839865''
    840866
    841 >>> f = TextFileForm(data={'description': u'Assistance'}, files={'file': {'filename': 'test3.txt', 'content': 'hello world'}}, instance=instance)
     867>>> f = TextFileForm(data={'description': u'Assistance'}, files={'file': SimpleUploadedFile('test3.txt', 'hello world')}, instance=instance)
    842868>>> f.is_valid()
    843869True
    844870>>> instance = f.save()
    845871>>> instance.file
    846872u'...test3.txt'
     873
     874# Delete the current file since this is not done by Django.
     875>>> os.unlink(instance.get_file_filename())
    847876>>> instance.delete()
    848877
     878>>> f = TextFileForm(data={'description': u'Assistance'}, files={'file': {'filename': 'test3.txt', 'content': 'hello world'}})
     879>>> f.is_valid()
     880True
     881>>> instance = f.save()
     882>>> instance.file
     883u'...test3.txt'
     884
     885# Delete the current file since this is not done by Django.
     886>>> os.unlink(instance.get_file_filename())
     887>>> instance.delete()
     888
    849889# ImageField ###################################################################
    850890
    851891# ImageField and FileField are nearly identical, but they differ slighty when
     
    858898
    859899>>> image_data = open(os.path.join(os.path.dirname(__file__), "test.png")).read()
    860900
     901>>> f = ImageFileForm(data={'description': u'An image'}, files={'image': SimpleUploadedFile('test.png', image_data)})
     902>>> f.is_valid()
     903True
     904>>> type(f.cleaned_data['image'])
     905<class 'django.newforms.fields.UploadedFile'>
     906>>> instance = f.save()
     907>>> instance.image
     908u'...test.png'
     909
     910# Delete the current file since this is not done by Django.
     911>>> os.unlink(instance.get_image_filename())
     912
    861913>>> f = ImageFileForm(data={'description': u'An image'}, files={'image': {'filename': 'test.png', 'content': image_data}})
    862914>>> f.is_valid()
    863915True
     
    885937
    886938# Override the file by uploading a new one.
    887939
    888 >>> f = ImageFileForm(data={'description': u'Changed it'}, files={'image': {'filename': 'test2.png', 'content': image_data}}, instance=instance)
     940>>> f = ImageFileForm(data={'description': u'Changed it'}, files={'image': SimpleUploadedFile('test2.png', image_data)}, instance=instance)
    889941>>> f.is_valid()
    890942True
    891943>>> instance = f.save()
    892944>>> instance.image
    893945u'...test2.png'
    894946
     947# Delete the current file since this is not done by Django.
     948>>> os.unlink(instance.get_image_filename())
    895949>>> instance.delete()
    896950
     951>>> f = ImageFileForm(data={'description': u'Changed it'}, files={'image': {'filename': 'test2.png', 'content': image_data}})
     952>>> f.is_valid()
     953True
     954>>> instance = f.save()
     955>>> instance.image
     956u'...test2.png'
     957
     958# Delete the current file since this is not done by Django.
     959>>> os.unlink(instance.get_image_filename())
     960>>> instance.delete()
     961
    897962# Test the non-required ImageField
    898963
    899964>>> f = ImageFileForm(data={'description': u'Test'})
     
    904969>>> instance.image
    905970''
    906971
    907 >>> f = ImageFileForm(data={'description': u'And a final one'}, files={'image': {'filename': 'test3.png', 'content': image_data}}, instance=instance)
     972>>> f = ImageFileForm(data={'description': u'And a final one'}, files={'image': SimpleUploadedFile('test3.png', image_data)}, instance=instance)
    908973>>> f.is_valid()
    909974True
    910975>>> instance = f.save()
    911976>>> instance.image
    912977u'...test3.png'
     978
     979# Delete the current file since this is not done by Django.
     980>>> os.unlink(instance.get_image_filename())
    913981>>> instance.delete()
    914982
     983>>> f = ImageFileForm(data={'description': u'And a final one'}, files={'image': {'filename': 'test3.png', 'content': image_data}})
     984>>> f.is_valid()
     985True
     986>>> instance = f.save()
     987>>> instance.image
     988u'...test3.png'
     989>>> instance.delete()
     990
    915991"""}
  • tests/regressiontests/bug639/tests.py

     
    99from regressiontests.bug639.models import Photo
    1010from django.http import QueryDict
    1111from django.utils.datastructures import MultiValueDict
     12from django.core.files.uploadedfile import SimpleUploadedFile
    1213
    1314class Bug639Test(unittest.TestCase):
    1415       
     
    2122       
    2223        # Fake a request query dict with the file
    2324        qd = QueryDict("title=Testing&image=", mutable=True)
    24         qd["image_file"] = {
    25             "filename" : "test.jpg",
    26             "content-type" : "image/jpeg",
    27             "content" : img
    28         }
    29        
     25        #qd["image_file"] = {
     26        #    "filename" : "test.jpg",
     27        #    "content-type" : "image/jpeg",
     28        #    "content" : img
     29        #    }
     30        qd["image_file"] = SimpleUploadedFile('test.jpg', img, 'image/jpeg')
     31
    3032        manip = Photo.AddManipulator()
    3133        manip.do_html2python(qd)
    3234        p = manip.save(qd)
     
    3941        Make sure to delete the "uploaded" file to avoid clogging /tmp.
    4042        """
    4143        p = Photo.objects.get()
    42         os.unlink(p.get_image_filename())
    43  No newline at end of file
     44        os.unlink(p.get_image_filename())
  • tests/regressiontests/forms/error_messages.py

     
    11# -*- coding: utf-8 -*-
    22tests = r"""
    33>>> from django.newforms import *
     4>>> from django.core.files.uploadedfile import SimpleUploadedFile
    45
    56# CharField ###################################################################
    67
     
    214215Traceback (most recent call last):
    215216...
    216217ValidationError: [u'INVALID']
    217 >>> f.clean({})
     218>>> f.clean(SimpleUploadedFile('name', None))
    218219Traceback (most recent call last):
    219220...
    220 ValidationError: [u'MISSING']
    221 >>> f.clean({'filename': 'name', 'content':''})
     221ValidationError: [u'EMPTY FILE']
     222>>> f.clean(SimpleUploadedFile('name', ''))
    222223Traceback (most recent call last):
    223224...
    224225ValidationError: [u'EMPTY FILE']
  • tests/regressiontests/forms/tests.py

     
    2626from regressions import tests as regression_tests
    2727from util import tests as util_tests
    2828from widgets import tests as widgets_tests
     29from warnings import filterwarnings
     30filterwarnings("ignore")
    2931
    3032__test__ = {
    3133    'extra_tests': extra_tests,
  • tests/regressiontests/forms/fields.py

     
    22tests = r"""
    33>>> from django.newforms import *
    44>>> from django.newforms.widgets import RadioFieldRenderer
     5>>> from django.core.files.uploadedfile import SimpleUploadedFile
    56>>> import datetime
    67>>> import time
    78>>> import re
     
    773774>>> f.clean({})
    774775Traceback (most recent call last):
    775776...
    776 ValidationError: [u'No file was submitted.']
     777ValidationError: [u'No file was submitted. Check the encoding type on the form.']
    777778
    778779>>> f.clean({}, '')
    779780Traceback (most recent call last):
    780781...
    781 ValidationError: [u'No file was submitted.']
     782ValidationError: [u'No file was submitted. Check the encoding type on the form.']
    782783
    783784>>> f.clean({}, 'files/test3.pdf')
    784785'files/test3.pdf'
     
    788789...
    789790ValidationError: [u'No file was submitted. Check the encoding type on the form.']
    790791
    791 >>> f.clean({'filename': 'name', 'content': None})
     792>>> f.clean(SimpleUploadedFile('name', None))
    792793Traceback (most recent call last):
    793794...
    794795ValidationError: [u'The submitted file is empty.']
    795796
    796 >>> f.clean({'filename': 'name', 'content': ''})
     797>>> f.clean(SimpleUploadedFile('name', ''))
    797798Traceback (most recent call last):
    798799...
    799800ValidationError: [u'The submitted file is empty.']
    800801
    801 >>> type(f.clean({'filename': 'name', 'content': 'Some File Content'}))
     802>>> type(f.clean(SimpleUploadedFile('name', 'Some File Content')))
    802803<class 'django.newforms.fields.UploadedFile'>
    803804
    804 >>> type(f.clean({'filename': 'name', 'content': 'Some File Content'}, 'files/test4.pdf'))
     805>>> type(f.clean(SimpleUploadedFile('name', 'Some File Content'), 'files/test4.pdf'))
    805806<class 'django.newforms.fields.UploadedFile'>
    806807
    807808# URLField ##################################################################
  • tests/regressiontests/forms/forms.py

     
    11# -*- coding: utf-8 -*-
    22tests = r"""
    33>>> from django.newforms import *
     4>>> from django.core.files.uploadedfile import SimpleUploadedFile
    45>>> import datetime
    56>>> import time
    67>>> import re
     
    14651466>>> print f
    14661467<tr><th>File1:</th><td><ul class="errorlist"><li>This field is required.</li></ul><input type="file" name="file1" /></td></tr>
    14671468
    1468 >>> f = FileForm(data={}, files={'file1': {'filename': 'name', 'content':''}}, auto_id=False)
     1469>>> f = FileForm(data={}, files={'file1': SimpleUploadedFile('name', '')}, auto_id=False)
    14691470>>> print f
    14701471<tr><th>File1:</th><td><ul class="errorlist"><li>The submitted file is empty.</li></ul><input type="file" name="file1" /></td></tr>
    14711472
     
    14731474>>> print f
    14741475<tr><th>File1:</th><td><ul class="errorlist"><li>No file was submitted. Check the encoding type on the form.</li></ul><input type="file" name="file1" /></td></tr>
    14751476
    1476 >>> f = FileForm(data={}, files={'file1': {'filename': 'name', 'content':'some content'}}, auto_id=False)
     1477>>> f = FileForm(data={}, files={'file1': SimpleUploadedFile('name', 'some content')}, auto_id=False)
    14771478>>> print f
    14781479<tr><th>File1:</th><td><input type="file" name="file1" /></td></tr>
    14791480>>> f.is_valid()
  • tests/regressiontests/test_client_regress/views.py

     
    11from django.contrib.auth.decorators import login_required
    22from django.http import HttpResponse, HttpResponseRedirect, HttpResponseServerError
     3import sha
    34
    45def no_template_view(request):
    56    "A simple view that expects a GET request, and returns a rendered template"
     
    1011    Check that a file upload can be updated into the POST dictionary without
    1112    going pear-shaped.
    1213    """
     14    from django.core.files.uploadedfile import UploadedFile
    1315    form_data = request.POST.copy()
    1416    form_data.update(request.FILES)
    15     if isinstance(form_data['file_field'], dict) and isinstance(form_data['name'], unicode):
     17    if isinstance(form_data.get('file_field'), UploadedFile) and isinstance(form_data['name'], unicode):
    1618        return HttpResponse('')
    1719    else:
    1820        return HttpResponseServerError()
    1921
     22def file_upload_view_verify(request):
     23    """
     24    Use the sha digest hash to verify the uploaded contents.
     25    """
     26    from django.core.files.uploadedfile import UploadedFile
     27    form_data = request.POST.copy()
     28    form_data.update(request.FILES)
     29
     30    # Check to see if unicode names worked out.
     31    if not request.FILES['file_unicode'].file_name.endswith(u'test_\u4e2d\u6587_Orl\xe9ans.jpg'):
     32        return HttpResponseServerError()
     33
     34    for key, value in form_data.items():
     35        if key.endswith('_hash'):
     36            continue
     37        if key + '_hash' not in form_data:
     38            continue
     39        submitted_hash = form_data[key + '_hash']
     40        if isinstance(value, UploadedFile):
     41            new_hash = sha.new(value.read()).hexdigest()
     42        else:
     43            new_hash = sha.new(value).hexdigest()
     44        if new_hash != submitted_hash:
     45            return HttpResponseServerError()
     46
     47    return HttpResponse('')
     48
    2049def get_view(request):
    2150    "A simple login protected view"
    2251    return HttpResponse("Hello world")
     
    3766def login_protected_redirect_view(request):
    3867    "A view that redirects all requests to the GET view"
    3968    return HttpResponseRedirect('/test_client_regress/get_view/')
    40 login_protected_redirect_view = login_required(login_protected_redirect_view)
    41  No newline at end of file
     69login_protected_redirect_view = login_required(login_protected_redirect_view)
  • tests/regressiontests/test_client_regress/models.py

     
    55from django.test import Client, TestCase
    66from django.core.urlresolvers import reverse
    77import os
     8import sha
    89
    910class AssertContainsTests(TestCase):
    1011    def test_contains(self):
     
    243244        response = self.client.post('/test_client_regress/file_upload/', post_data)
    244245        self.assertEqual(response.status_code, 200)
    245246
     247    def test_large_upload(self):
     248        import tempfile
     249        dir = tempfile.gettempdir()
     250
     251        (fd, name1) = tempfile.mkstemp(suffix='.file1', dir=dir)
     252        file1 = os.fdopen(fd, 'w+b')
     253        file1.write('a' * (2 ** 21))
     254        file1.seek(0)
     255
     256        (fd, name2) = tempfile.mkstemp(suffix='.file2', dir=dir)
     257        file2 = os.fdopen(fd, 'w+b')
     258        file2.write('a' * (10 * 2 ** 20))
     259        file2.seek(0)
     260
     261        # This file contains chinese symbols for a name.
     262        name3 = os.path.join(dir, u'test_&#20013;&#25991;_Orl\u00e9ans.jpg')
     263        file3 = open(name3, 'w+b')
     264        file3.write('b' * (2 ** 10))
     265        file3.seek(0)
     266
     267        post_data = {
     268            'name': 'Ringo',
     269            'file_field1': file1,
     270            'file_field2': file2,
     271            'file_unicode': file3,
     272            }
     273
     274        for key in post_data.keys():
     275            try:
     276                post_data[key + '_hash'] = sha.new(post_data[key].read()).hexdigest()
     277                post_data[key].seek(0)
     278            except AttributeError:
     279                post_data[key + '_hash'] = sha.new(post_data[key]).hexdigest()
     280
     281        response = self.client.post('/test_client_regress/file_upload_verify/', post_data)
     282
     283        for name in (name1, name2, name3):
     284            try:
     285                os.unlink(name)
     286            except:
     287                pass
     288
     289        self.assertEqual(response.status_code, 200)
     290
     291
    246292class LoginTests(TestCase):
    247293    fixtures = ['testdata']
    248294
  • tests/regressiontests/test_client_regress/urls.py

     
    44urlpatterns = patterns('',
    55    (r'^no_template_view/$', views.no_template_view),
    66    (r'^file_upload/$', views.file_upload_view),
     7    (r'^file_upload_verify/$', views.file_upload_view_verify),
    78    (r'^get_view/$', views.get_view),
    89    url(r'^arg_view/(?P<name>.+)/$', views.view_with_argument, name='arg_view'),
    910    (r'^login_protected_redirect_view/$', views.login_protected_redirect_view)
  • tests/regressiontests/datastructures/tests.py

     
    117117>>> d['person']['2']['firstname']
    118118['Adrian']
    119119
    120 ### FileDict ################################################################
    121 
    122 >>> d = FileDict({'content': 'once upon a time...'})
     120### ImmutableList ################################################################
     121>>> d = ImmutableList(range(10))
     122>>> d.sort()
     123Traceback (most recent call last):
     124  File "<stdin>", line 1, in <module>
     125  File "/var/lib/python-support/python2.5/django/utils/datastructures.py", line 359, in complain
     126    raise AttributeError, self.warning
     127AttributeError: ImmutableList object is immutable.
    123128>>> repr(d)
    124 "{'content': '<omitted>'}"
    125 >>> d = FileDict({'other-key': 'once upon a time...'})
    126 >>> repr(d)
    127 "{'other-key': 'once upon a time...'}"
     129'(0, 1, 2, 3, 4, 5, 6, 7, 8, 9)'
     130>>> d = ImmutableList(range(10), warning="Object is immutable!")
     131>>> d[1]
     1321
     133>>> d[1] = 'test'
     134Traceback (most recent call last):
     135  File "<stdin>", line 1, in <module>
     136  File "/var/lib/python-support/python2.5/django/utils/datastructures.py", line 359, in complain
     137    raise AttributeError, self.warning
     138AttributeError: Object is immutable!
    128139"""
  • AUTHORS

     
    5858    Jökull Sólberg Auðunsson <jokullsolberg@gmail.com>
    5959    Arthur <avandorp@gmail.com>
    6060    David Avsajanishvili <avsd05@gmail.com>
    61     axiak@mit.edu
     61    Mike Axiak <axiak@mit.edu>
    6262    Niran Babalola <niran@niran.org>
    6363    Morten Bagai <m@bagai.com>
    6464    Mikaël Barbero <mikael.barbero nospam at nospam free.fr>
     
    135135    Marc Fargas <telenieko@telenieko.com>
    136136    Szilveszter Farkas <szilveszter.farkas@gmail.com>
    137137    favo@exoweb.net
     138    fdr <drfarina@gmail.com>
    138139    Dmitri Fedortchenko <zeraien@gmail.com>
    139140    Bill Fenner <fenner@gmail.com>
    140141    Stefane Fermgier <sf@fermigier.com>
  • docs/upload_handling.txt

     
     1============
     2File Uploads
     3============
     4
     5**New in Django development version**
     6
     7Most Web sites wouldn't be complete without a way to upload files. Before the
     8file even gets into ``request.FILES`` (discussed in the `Request
     9and response objects`_ documentation) Django has to decide where to put the
     10all the incoming data. This document describes how Django deals with this
     11stage of the upload process.
     12
     13
     14.. _Request and response objects: ../request_response/#attributes
     15
     16Default Behavior
     17================
     18
     19If you set up a quick model that contains a `FileField`_ and use the `Admin
     20interface`_ you can quickly create a form to upload files to your site.
     21By default, if the upload is smaller than *2.5 Megabytes* in size, Django
     22will hold the entire contents of the upload in memory, and the file will
     23quickly be saved to its final destination (as defined by
     24`settings.MEDIA_ROOT`_) without any intermediary.
     25
     26If the entire upload is larger than 2.5 Megabytes, then it will -- by
     27default -- write the contents of the uploaded file to a temporary file
     28in your operating system's default temporary directory. On a posix platform,
     29this means that you could expect a file similar to ``/tmp/tmpzfp6I6.upload``
     30to be generated and filled during the upload process.
     31
     32.. note::
     33    You may find that the default temporary directory is not writable.
     34    This is especially true if you are on shared hosting. If this is
     35    the case, Django will raise an error when you try to upload. Please
     36    read below in `Extending the Default`_ to change this directory.
     37
     38
     39.. _FileField: ../model-api/#filefield
     40.. _Admin interface: ../tutorial02/#activate-the-admin-site
     41.. _settings.MEDIA_ROOT: ../settings/#media-root
     42
     43Extending the Default
     44=====================
     45
     46If you have quite a bit of memory, you may decide that you want Django to
     47stream only if the entire upload exceeds 10 Megabytes. You may even
     48decide that whatever directory is the platform default is not suitable
     49for your project. If this is the case, Django provides these two settings:
     50
     51    =============================== ======================================
     52    Setting                         Description
     53    =============================== ======================================
     54    ``FILE_UPLOAD_MAX_MEMORY_SIZE`` The maximum size of a request in bytes
     55                                    for which Django will try to load the
     56                                    entire upload contents in memory.
     57
     58    ``FILE_UPLOAD_TEMP_DIR``        The directory on the file system where
     59                                    uploaded contents will be temporarily
     60                                    stored if not completely in memory.
     61                                    E.g.: ``"/tmp"``
     62    =============================== ======================================
     63
     64There is one final setting -- ``FILE_UPLOAD_HANDLERS`` -- which allows complete
     65customization of the upload process.
     66
     67Upload Handlers
     68===============
     69
     70Through upload handlers Django provides the flexibility to extend the
     71upload process beyond simple storage. You can use custom handlers to enforce
     72user-level quotas, compress data on the fly, render progress bars, and
     73even send data to another warehouse directly without storing it locally!
     74
     75There are two pieces to the Django upload handling: the upload handler and the
     76uploaded file. These are both represented by python classes --
     77``FileUploadHandler`` and ``UploadedFile`` respectively. Throughout the lifetime
     78of the upload process, Django will call on the upload handler to handle the
     79upload, while the upload handler is expected to return an ``UploadedFile``
     80object at the end of a file's upload.
     81
     82Setting Default Upload Handlers for your Project
     83------------------------------------------------
     84
     85Similar to `Middleware`_, upload handlers have an order that is initially
     86defined in ``settings.FILE_UPLOAD_HANDLERS``. The default value is::
     87
     88    ("django.core.files.fileuploadhandler.MemoryFileUploadHandler",
     89     "django.core.files.fileuploadhandler.TemporaryFileUploadHandler",)
     90
     91This literally means: Try putting the upload in memory first and failing
     92that put the upload in a temporary file. However, this behavior is
     93completely dependent on how each of those handlers is defined. For instance,
     94just because there isn't an example, there's no reason an upload handler
     95could run first recording information.
     96
     97.. _Middleware: ../middleware/
     98
     99Modifying your Upload Handlers Dynamically
     100------------------------------------------
     101
     102During the lifetime of your project, you may realize that a particular
     103view or views require different uploading behavior. For this reason,
     104the ``request`` object contains a list of handlers
     105(``request.upload_handlers``) that will be called in order. To append
     106a handler to the list, you would append to it like any other list.
     107For example, suppose you had an ``ProgressBarUploadHandler`` class.
     108To append it to your upload handlers you would write::
     109
     110    request.upload_handlers.append(ProgressBarUploadHandler())
     111
     112However, since the progress bar handler would probably need to run before the
     113other handlers get a chance, you'd probably want to insert it. That is, you'd
     114write::
     115
     116   request.upload_handlers.insert(0, ProgressBarUploadHandler())
     117
     118If you want to replace the upload handlers completely, you can just assign a new
     119list::
     120
     121   request.upload_handlers = [ProgressBarUploadHandler()]
     122
     123And all Django will do is keep a progress, but do nothing with the file itself!
     124One more note: After the upload has completed, you are no longer allowed to
     125assign or modify this list, and Django will raise an error if you try to modify
     126this list after an upload.
     127
     128Writing a File Upload Handler
     129-----------------------------
     130
     131All file upload handlers are subclasses of ``FileUploadHandler``, found in
     132``django.core.files.fileuploadhandler``. To create your handler, you need to
     133define the required methods followed by the methods that make most sense to you:
     134
     135chunk_size
     136~~~~~~~~~~
     137
     138This is an integer attribute that specifies how large the chunks we should put
     139into memory from the network are. The chunk sizes should be divisible by ``4``
     140and should not exceed ``2 ** 31 - 1`` in size. When there are multiple chunk
     141sizes provided by multiple handlers, Django will use the smallest chunk size.
     142
     143new_file
     144~~~~~~~~
     145
     146This signals that a new file is starting. You can initialize a file or whatever
     147else is needed on a new file upload.
     148
     149Interface: ``new_file(self, field_name, file_name, content_type, content_length,
     150charset)``
     151
     152``field_name`` is a string name of the field this was POSTed as.
     153
     154``file_name`` is the unicode filename that was provided by the browser.
     155
     156``content_type`` is the MIME type provided by the browser -- E.g.
     157``'image/jpeg'``.
     158
     159``content_length`` is the length of the image given by the browser if provided,
     160``None`` otherwise.
     161
     162``charset`` is the charset given by the browser if provided, ``None`` otherwise.
     163
     164Returns: ``None`` if you want other handlers to get ``new_file`` called.
     165Something nonzero if you don't want subsequent handlers to get a chance.
     166
     167receive_data_chunk
     168~~~~~~~~~~~~~~~~~~
     169*required*
     170
     171This method is used to do something with the new chunk of data. For example: The
     172``TemporaryFileUploadHandler`` takes the data and writes it to disk.
     173
     174Interface: ``receive_data_chunk(self, raw_data, start, stop)``
     175
     176``raw_data`` is a byte string containing the uploaded data.
     177
     178``start`` is the byte number where the chunk starts.
     179
     180``stop`` is the byte number where the chunk stops.
     181
     182Returns: ``None`` if you don't want the subsequent upload handlers to receive
     183the data. Whatever else you return gets fed into the subsequent upload handlers'
     184``receive_data_chunk`` method.
     185
     186Exceptions: If you raise a ``StopUpload`` or a ``SkipFile`` exception, the
     187upload will abort or the file will be skipped respectively.
     188
     189file_complete
     190~~~~~~~~~~~~~
     191*required*
     192
     193This method defines when a function has finished uploading and gets packaged
     194into an ``UploadedFile`` object.
     195
     196Interface: ``file_complete(self, file_size)``
     197
     198``file_size`` is the number of bytes you have received for this file.
     199
     200Returns: An ``UploadedFile`` object to set in the ``request.FILES`` dictionary.
     201``None`` if you want the subsequent upload handlers to get an ``UploadedFile``
     202object.
     203
     204upload_complete
     205~~~~~~~~~~~~~~~
     206
     207This method defines when the entire upload has completed.
     208
     209Interface: ``upload_complete(self)``
     210
     211handle_raw_input
     212~~~~~~~~~~~~~~~~
     213
     214This method allows the handler to completely override the parsing of the raw
     215HTTP-layered input.
     216
     217Interface: ``handle_raw_input(self, input_data, META, content_length, boundary,
     218encoding)``
     219
     220``input_data`` is a file-like object that supports the read operation.
     221
     222``META`` is the same object as ``request.META``.
     223
     224``content_length`` is the length of the data in ``input_data``. Don't read more
     225than ``content_length`` bytes from ``input_data``.
     226
     227``boundary`` is the MIME boundary for this request.
     228
     229``encoding`` is the encoding of the request.
     230
     231Returns: ``None`` if you want to go on to the next stage, ``(POST, FILES)`` if
     232you want to return the new data structures suitable for the request directly.
     233
     234Defining an Uploaded File
     235-------------------------
     236
     237All file upload handlers are subclasses of ``UploadedFile``, found in
     238``django.core.files.uploadedfile``. The uploaded file object is returned by the
     239handler above in ``file_complete``. To create your own uploaded file class, you
     240need to define the required methods followed by the methods that make most sense
     241to you:
     242
     243read
     244~~~~
     245*required*
     246
     247Interface: ``read(self, num_bytes=None)``
     248
     249Returns: A byte string of length ``num_bytes`` (or the size of the file if
     250``num_bytes`` is not supplied).
     251
     252chunk
     253~~~~~
     254
     255A generator to yield small chunks from the file. With the ``read()`` defined,
     256the ``UploadedFile`` class already defines a ``chunk()`` method that's probably
     257suitable.
     258
     259Interface: ``chunk(self, chunk_size=None)``
     260
     261multiple_chunks
     262~~~~~~~~~~~~~~~
     263
     264Interface: ``multiple_chunks(self, chunk_size=None)``
     265
     266Returns: ``True`` or ``False`` depending on whether or not the user of this file
     267can expect more than one chunk when calling ``chunk(self, chunk_size)``. If all
     268the data is in memory, you should return ``False``.
     269
     270temporary_file_path
     271~~~~~~~~~~~~~~~~~~~
     272
     273If defined, this method should return the file path of the file on the operating
     274system. This will let users move files rather than write to new files if the
     275option is available.
     276
     277Interface: ``temporary_file_path(self)``
     278
     279Returns: A file path in a file system on the local computer.
     280
     281upload_errors
     282~~~~~~~~~~~~~
     283
     284If defined, this method should return a string describing errors that occured
     285during uploads.
     286
     287Interface: ``upload_errors(self)``
     288
     289Returns: A unicode string describing an error that occured with the file's upload.
  • docs/db-api.txt

     
    13061306Using raw strings (e.g., ``r'foo'`` instead of ``'foo'``) for passing in the
    13071307regular expression syntax is recommended.
    13081308
     1309Regular expression matching is not supported on the ``ado_mssql`` backend.
     1310It will raise a ``NotImplementedError`` at runtime.
     1311
    13091312iregex
    13101313~~~~~~
    13111314
  • docs/settings.txt

     
    279279
    280280The database backend to use. The build-in database backends are
    281281``'postgresql_psycopg2'``, ``'postgresql'``, ``'mysql'``, ``'mysql_old'``,
    282 ``'sqlite3'`` and ``'oracle'``.
     282``'sqlite3'``, ``'oracle'``, or ``'ado_mssql'``.
    283283
    284284In the Django development version, you can use a database backend that doesn't
    285285ship with Django by setting ``DATABASE_ENGINE`` to a fully-qualified path (i.e.
     
    513513The character encoding used to decode any files read from disk. This includes
    514514template files and initial SQL data files.
    515515
     516FILE_UPLOAD_HANDLERS
     517--------------------
     518
     519**New in Django development version**
     520
     521Default::
     522
     523    ("django.core.files.fileuploadhandler.MemoryFileUploadHandler",
     524     "django.core.files.fileuploadhandler.TemporaryFileUploadHandler",)
     525
     526A tuple of handlers to use for uploading.
     527
     528FILE_UPLOAD_MAX_MEMORY_SIZE
     529---------------------------
     530
     531**New in Django development version**
     532
     533Default: ``2621440``
     534
     535The maximum size (in bytes) that an upload will be before it gets streamed to the file system.
     536
     537FILE_UPLOAD_TEMP_DIR
     538--------------------
     539
     540**New in Django development version**
     541
     542Default: ``None``
     543
     544The directory to store data temporarily while uploading files. If ``None``, Django will use the standard temporary directory for the operating system. For example, this will default to '/tmp' on *nix-style operating systems.
     545
    516546FIXTURE_DIRS
    517547-------------
    518548
  • docs/newforms.txt

     
    805805need to bind the file data containing the mugshot image::
    806806
    807807    # Bound form with an image field
     808    >>> from django.core.files.uploadedfile import SimpleUploadedFile
    808809    >>> data = {'subject': 'hello',
    809810    ...         'message': 'Hi there',
    810811    ...         'sender': 'foo@example.com',
    811812    ...         'cc_myself': True}
    812     >>> file_data = {'mugshot': {'filename':'face.jpg'
    813     ...                          'content': <file data>}}
     813    >>> file_data = {'mugshot': SimpleUploadedFile('face.jpg', <file data>)}
    814814    >>> f = ContactFormWithMugshot(data, file_data)
    815815
    816816In practice, you will usually specify ``request.FILES`` as the source
Back to Top