Ticket #2070: 2070_revision7363.diff

File 2070_revision7363.diff, 93.2 KB (added by axiak, 7 years ago)

Altered documentation to be more approachable.

  • django/http/multipartparser.py

     
     1"""
     2MultiPart parsing for file uploads.
     3
     4This object will take the file upload headers
     5and the file upload handler and chunk the upload
     6data for the handler to deal with.
     7"""
     8from django.utils.datastructures import MultiValueDict
     9from django.utils.encoding import force_unicode
     10from django.utils.text import unescape_entities
     11
     12__all__ = ('MultiPartParser','MultiPartParserError','InputStreamExhausted')
     13
     14class MultiPartParserError(Exception):
     15    pass
     16
     17class InputStreamExhausted(Exception):
     18    """ No more reads are allowed from this device. """
     19    pass
     20
     21class MultiPartParser(object):
     22    """
     23    A rfc2388 multipart/form-data parser.
     24
     25    parse() reads the input stream in chunk_size chunks and returns a
     26    tuple of (POST MultiValueDict, FILES MultiValueDict). If
     27    file_upload_dir is defined files will be streamed to temporary
     28    files in the specified directory.
     29    """
     30    def __init__(self, META, input_data, upload_handlers, encoding=None):
     31        """
     32        Initialize the MultiPartParser object.
     33
     34        *META* -- The standard META dictionary in Django request objects.
     35        *input_data* -- The raw post data, as a bytestring.
     36        *upload_handler* -- An object of type UploadHandler
     37                            that performs operations on the uploaded
     38                            data.
     39        *encoding* -- The encoding with which to treat the incoming data.
     40        """
     41        # Import cgi utilities for (near) future use.
     42        global valid_boundary, settings
     43        from django.conf import settings
     44        from cgi import valid_boundary
     45
     46        #
     47        # Content-Type should containt multipart and the boundary information.
     48        #
     49
     50        content_type = META.get('HTTP_CONTENT_TYPE', META.get('CONTENT_TYPE', ''))
     51        if not content_type.startswith('multipart/'):
     52            raise MultiPartParserError('Invalid Content-Type: %s' %
     53                                       content_type)
     54
     55        # Parse the header to get the boundary to split the parts.
     56        ctypes, opts = parse_header(content_type)
     57        boundary = opts.get('boundary')
     58        if not boundary or not valid_boundary(boundary):
     59            raise MultiPartParserError('Invalid boundary in multipart: %s' %
     60                                       boundary)
     61
     62
     63        #
     64        # Content-Length should contain the length of the body we are about
     65        # to receive.
     66        #
     67        try:
     68            content_length = int(META.get('HTTP_CONTENT_LENGTH',
     69                                          META.get('CONTENT_LENGTH',0)))
     70        except (ValueError, TypeError):
     71            # For now set it to 0...we'll try again later on down.
     72            content_length = 0
     73
     74        if content_length <= 0:
     75            # This means we shouldn't continue...raise an error.
     76            raise MultiPartParserError("Invalid content length: %r" % content_length)
     77
     78        self._boundary = boundary
     79        self._input_data = input_data
     80
     81        # For compatibility with low-level network APIs (with 32-bit integers),
     82        # the chunk size should be < 2^31, but still divisible by 4.
     83        self._chunk_size = min(2147483644, *[x.chunk_size for x in upload_handlers
     84                                            if x.chunk_size])
     85
     86        self._meta = META
     87        self._encoding = encoding or settings.DEFAULT_CHARSET
     88        self._content_length = content_length
     89        self._upload_handlers = upload_handlers
     90
     91    def parse(self):
     92        """
     93        Parse the POST data and break it into a FILES MultiValueDict
     94        and a POST MultiValueDict.
     95
     96           *returns* -- A tuple containing the POST and FILES dictionary,
     97                        respectively.
     98        """
     99        from django.core.files.fileuploadhandler import StopUpload, SkipFile
     100        from django.http import QueryDict
     101
     102        encoding = self._encoding
     103        handlers = self._upload_handlers
     104
     105        limited_input_data = LimitBytes(self._input_data, self._content_length)
     106
     107        # See if the handler will want to take care of the parsing.
     108        # This allows overriding everything if somebody wants it.
     109        for handler in handlers:
     110            result = handler.handle_raw_input(limited_input_data,
     111                                              self._meta,
     112                                              self._content_length,
     113                                              self._boundary,
     114                                              encoding)
     115            if result is not None:
     116                return result[0], result[1]
     117
     118        # Create the data structures to be used later.
     119        self._post = QueryDict('', mutable=True)
     120        self._files = MultiValueDict()
     121
     122        # Instantiate the parser and stream:
     123        stream = LazyStream(ChunkIter(limited_input_data, self._chunk_size))
     124
     125        # Whether or not to signal a file-completion at the beginning of the loop.
     126        old_field_name = None
     127        counters = [0] * len(handlers)
     128
     129        for item_type, meta_data, stream in Parser(stream, self._boundary):
     130            if old_field_name:
     131                # We run this at the beginning of the next loop
     132                # since we cannot be sure a file is complete until
     133                # we hit the next boundary/part of the multipart content.
     134                self.handle_file_complete(old_field_name, counters)
     135
     136            try:
     137                disposition = meta_data['content-disposition'][1]
     138                field_name = disposition['name'].strip()
     139            except (KeyError, IndexError, AttributeError):
     140                continue
     141
     142            transfer_encoding = meta_data.get('content-transfer-encoding')
     143            field_name = force_unicode(field_name, encoding, errors='replace')
     144
     145            if item_type == 'FIELD':
     146                # This is a post field, we can just set it in the post
     147                if transfer_encoding == 'base64':
     148                    raw_data = stream.read()
     149                    try:
     150                        data = str(raw_data).decode('base64')
     151                    except:
     152                        data = raw_data
     153                else:
     154                    data = stream.read()
     155
     156                self._post.appendlist(field_name,
     157                                      force_unicode(data, encoding, errors='replace'))
     158            elif item_type == 'FILE':
     159                # This is a file, use the handler...
     160                file_successful = True
     161                file_name = disposition.get('filename')
     162                if not file_name:
     163                    continue
     164                file_name = force_unicode(file_name, encoding, errors='replace')
     165                file_name = self.IE_sanitize(unescape_entities(file_name))
     166
     167
     168                content_type = meta_data.get('content-type', ('',))[0].strip()
     169                try:
     170                    charset = meta_data.get('content-type', (0,{}))[1].get('charset', None)
     171                except:
     172                    charset = None
     173
     174                try:
     175                    content_length = int(meta_data.get('content-length')[0])
     176                except (IndexError, TypeError, ValueError):
     177                    content_length = None
     178
     179                counters = [0] * len(handlers)
     180                try:
     181                    for handler in handlers:
     182                        retval = handler.new_file(field_name, file_name,
     183                                                  content_type, content_length,
     184                                                  charset)
     185                        if retval:
     186                            break
     187
     188                    for chunk in stream:
     189                        if transfer_encoding == 'base64':
     190                            # We only special-case base64 transfer encoding
     191                            try:
     192                                chunk = str(chunk).decode('base64')
     193                            except Exception, e:
     194                                # Since this is only a chunk, any error is an unfixable error.
     195                                raise MultiValueParseError("Could not decode base64 data: %r" % e)
     196
     197                        for i, handler in enumerate(handlers):
     198                            chunk_length = len(chunk)
     199                            counters[i] += chunk_length
     200                            chunk = handler.receive_data_chunk(chunk,
     201                                                               counters[i] - chunk_length,
     202                                                               counters[i])
     203                            if chunk is None:
     204                                # If the chunk received by the handler is None, then don't continue.
     205                                break
     206
     207                except (StopUpload, SkipFile), e:
     208                    file_successful = False
     209                    if isinstance(e, SkipFile):
     210                        # Just use up the rest of this file...
     211                        stream.exhaust()
     212                    elif isinstance(e, StopUpload):
     213                        # Abort the parsing and break
     214                        parser.abort()
     215                        break
     216                else:
     217                    # Handle file upload completions on next iteration.
     218                    old_field_name = field_name
     219            else:
     220                # If this is neither a FIELD or a FILE, just exhaust the stream.
     221                stream.exhuast()
     222
     223        # Make sure that the request data is all fed
     224        limited_input_data.exhaust()
     225        # Signal that the upload has completed.
     226        for handler in handlers:
     227            retval = handler.upload_complete()
     228            if retval:
     229                break
     230
     231        return self._post, self._files
     232
     233    def handle_file_complete(self, old_field_name, counters):
     234        """
     235        Handle all the signalling that takes place when a file is complete.
     236        """
     237        for i, handler in enumerate(self._upload_handlers):
     238            file_obj = handler.file_complete(counters[i])
     239            if file_obj:
     240                # If it returns a file object, then set the files dict.
     241                self._files.appendlist(force_unicode(old_field_name,
     242                                                     self._encoding,
     243                                                     errors='replace'),
     244                                       file_obj)
     245                break
     246
     247    def IE_sanitize(self, filename):
     248        """cleanup filename from IE full paths"""
     249        return filename and filename[filename.rfind("\\")+1:].strip()
     250
     251
     252class LazyStream(object):
     253    """
     254    The LazyStream wrapper allows one to pull and "unget" bytes from a stream.
     255
     256    Given a producer object (an iterator that yields bytestrings), the
     257    LazyStream object will support iteration, reading, and keeping a
     258    "look-back" variable in case you need to "unget" some bytes.
     259    """
     260    def __init__(self, producer, length=None):
     261        """
     262        Every LazyStream must have a producer when instantiated.
     263
     264        A producer is an iterable that returns a string each time it
     265        is called.
     266        """
     267        self._producer = producer
     268        self._empty = False
     269        self._leftover = ''
     270        self.length = length
     271        self.position = 0
     272        self._remaining = length
     273
     274    def tell(self):
     275        return self.position
     276
     277    def read(self, size=None):
     278        def parts():
     279            remaining = (size is not None and [size] or [self._remaining])[0]
     280            # do the whole thing in one shot if no limit was provided.
     281            if remaining is None:
     282                yield ''.join(self)
     283                return
     284
     285            # otherwise do some bookkeeping to return exactly enough
     286            # of the stream and stashing any extra content we get from
     287            # the producer
     288            while remaining != 0:
     289                assert remaining > 0, 'remaining bytes to read should never go negative'
     290
     291                chunk = self.next()
     292
     293                emitting = chunk[:remaining]
     294                self.unget(chunk[remaining:])
     295                remaining -= len(emitting)
     296                yield emitting
     297
     298        out = ''.join(parts())
     299        self.position += len(out)
     300        return out
     301
     302    def next(self):
     303        """
     304        Used when the exact number of bytes to read is unimportant.
     305
     306        This procedure just returns whatever is chunk is conveniently
     307        returned from the iterator instead. Useful to avoid
     308        unnecessary bookkeeping if performance is an issue.
     309        """
     310        if self._leftover:
     311            output = self._leftover
     312            self.position += len(output)
     313            self._leftover = ''
     314            return output
     315        else:
     316            output = self._producer.next()
     317            self.position += len(output)
     318            return output
     319
     320    def close(self):
     321        """
     322        Used to invalidate/disable this lazy stream.
     323
     324        Replaces the producer with an empty list. Any leftover bytes
     325        that have already been read will still be reported upon read()
     326        and/or next().
     327        """
     328        self._producer = []
     329
     330    def __iter__(self):
     331        return self
     332
     333    def unget(self, bytes):
     334        """
     335        Places bytes back onto the front of the lazy stream.
     336
     337        Future calls to read() will return those bytes first. The
     338        stream position and thus tell() will be rewound.
     339        """
     340        self.position -= len(bytes)
     341        self._leftover = ''.join([bytes, self._leftover])
     342
     343    def exhaust(self):
     344        """
     345        Exhausts the entire underlying stream.
     346
     347        Useful for skipping and advancing sections.
     348        """
     349        for thing in self:
     350            pass
     351
     352
     353class ChunkIter(object):
     354    """
     355    An iterable that will yield chunks of data.
     356    Given a file-like object as the constructor,
     357    this object will yield chunks of read operations
     358    from that object.
     359    """
     360    def __init__(self, flo, chunk_size=64 * 1024):
     361        self.flo = flo
     362        self.chunk_size = chunk_size
     363
     364    def next(self):
     365        try:
     366            data = self.flo.read(self.chunk_size)
     367        except InputStreamExhausted:
     368            raise StopIteration
     369        if data:
     370            return data
     371        else:
     372            raise StopIteration
     373
     374    def __iter__(self):
     375        return self
     376
     377
     378class LimitBytes(object):
     379    """ Limit bytes for a file object. """
     380    def __init__(self, fileobject, length):
     381        self._file = fileobject
     382        self.remaining = length
     383
     384    def read(self, num_bytes=None):
     385        """
     386        Read data from the underlying file.
     387        If you ask for too much or there isn't anything left,
     388        this will raise an InputStreamExhausted error.
     389        """
     390        if self.remaining <= 0:
     391            raise InputStreamExhausted()
     392        if num_bytes is None:
     393            num_bytes = self.remaining
     394        else:
     395            num_bytes = min(num_bytes, self.remaining)
     396        self.remaining -= num_bytes
     397        return self._file.read(num_bytes)
     398
     399    def exhaust(self):
     400        """
     401        Exhaust this file until all of the bytes it was limited by
     402        have been read.
     403        """
     404        while self.remaining > 0:
     405            num_bytes = min(self.remaining, 16384)
     406            __ = self._file.read(num_bytes)
     407            self.remaining -= num_bytes
     408
     409
     410class InterBoundaryIter(object):
     411    """
     412    A Producer that will iterate over boundaries.
     413    """
     414    def __init__(self, stream, boundary):
     415        self._stream = stream
     416        self._boundary = boundary
     417
     418    def __iter__(self):
     419        return self
     420
     421    def next(self):
     422        try:
     423            return LazyStream(BoundaryIter(self._stream, self._boundary))
     424        except InputStreamExhausted:
     425            raise StopIteration
     426
     427
     428class BoundaryIter(object):
     429    """
     430    A Producer that is sensitive to boundaries.
     431
     432    Will happily yield bytes until a boundary is found. Will yield the
     433    bytes before the boundary, throw away the boundary bytes
     434    themselves, and push the post-boundary bytes back on the stream.
     435
     436    The future calls to .next() after locating the boundary will raise
     437    a StopIteration exception.
     438    """
     439
     440    def __init__(self, stream, boundary):
     441        self._stream = stream
     442        self._boundary = boundary
     443        self._done = False
     444        # rollback an additional six bytes because the format is like
     445        # this: CRLF<boundary>[--CRLF]
     446        self._rollback = len(boundary) + 6
     447
     448        # Try to use mx fast string search if available. Otherwise
     449        # use Python find. Wrap the latter for consistency.
     450        unused_char = self._stream.read(1)
     451        if not unused_char:
     452            raise InputStreamExhausted
     453        self._stream.unget(unused_char)
     454        try:
     455            from mx.TextTools import FS
     456            self._fs = FS(boundary).find
     457        except ImportError:
     458            self._fs = lambda data: data.find(boundary)
     459
     460    def __iter__(self):
     461        return self
     462
     463    def next(self):
     464        if self._done:
     465            raise StopIteration
     466
     467        stream = self._stream
     468        rollback = self._rollback
     469
     470        bytes_read = 0
     471        chunks = []
     472        for bytes in stream:
     473            bytes_read += len(bytes)
     474            chunks.append(bytes)
     475            if bytes_read > rollback:
     476                break
     477            if not bytes:
     478                break
     479        else:
     480            self._done = True
     481
     482        if not chunks:
     483            raise StopIteration
     484
     485        chunk = ''.join(chunks)
     486        boundary = self._find_boundary(chunk, len(chunk) < self._rollback)
     487
     488        if boundary:
     489            end, next = boundary
     490            stream.unget(chunk[next:])
     491            self._done = True
     492            return chunk[:end]
     493        else:
     494            # make sure we dont treat a partial boundary (and
     495            # its separators) as data
     496            if not chunk[:-rollback]:# and len(chunk) >= (len(self._boundary) + 6):
     497                # There's nothing left, we should just return and mark as done.
     498                self._done = True
     499                return chunk
     500            else:
     501                stream.unget(chunk[-rollback:])
     502                return chunk[:-rollback]
     503
     504    def _find_boundary(self, data, eof = False):
     505        """
     506        Finds a multipart boundary in data.
     507
     508        Should no boundry exist in the data None is returned
     509        instead. Otherwise a tuple containing
     510        the indices of the following are returned:
     511
     512         * the end of current encapsulation
     513
     514         * the start of the next encapsulation
     515        """
     516        index = self._fs(data)
     517        if index < 0:
     518            return None
     519        else:
     520            end = index
     521            next = index + len(self._boundary)
     522            data_len = len(data) - 1
     523            # backup over CRLF
     524            if data[max(0,end-1)] == '\n':
     525                end -= 1
     526            if data[max(0,end-1)] == '\r':
     527                end -= 1
     528            # skip over --CRLF
     529            #if data[min(data_len,next)] == '-':
     530            #    next += 1
     531            #if data[min(data_len,next)] == '-':
     532            #    next += 1
     533            #if data[min(data_len,next)] == '\r':
     534            #    next += 1
     535            #if data[min(data_len,next)] == '\n':
     536            #    next += 1
     537            return end, next
     538
     539
     540def ParseBoundaryStream(stream, max_header_size):
     541        """
     542        Parses one and exactly one stream that encapsulates a boundary.
     543        """
     544        # Stream at beginning of header, look for end of header
     545        # and parse it if found. The header must fit within one
     546        # chunk.
     547        chunk = stream.read(max_header_size)
     548        # 'find' returns the top of these four bytes, so we'll
     549        # need to munch them later to prevent them from polluting
     550        # the payload.
     551        header_end = chunk.find('\r\n\r\n')
     552
     553        def _parse_header(line):
     554            main_value_pair, params = parse_header(line)
     555            try:
     556                name, value = main_value_pair.split(':', 1)
     557            except:
     558                raise ValueError("Invalid header: %r" % line)
     559            return name, (value, params)
     560
     561        if header_end == -1:
     562            # we find no header, so we just mark this fact and pass on
     563            # the stream verbatim
     564            stream.unget(chunk)
     565            return ('RAW', {}, stream)
     566
     567        header = chunk[:header_end]
     568
     569        # here we place any excess chunk back onto the stream, as
     570        # well as throwing away the CRLFCRLF bytes from above.
     571        stream.unget(chunk[header_end + 4:])
     572
     573        TYPE = 'RAW'
     574        outdict = {}
     575
     576        # Eliminate blank lines
     577        for line in header.split('\r\n'):
     578            # This terminology ("main value" and "dictionary of
     579            # parameters") is from the Python docs.
     580            try:
     581                name, (value, params) = _parse_header(line)
     582            except:
     583                continue
     584
     585            if name == 'content-disposition':
     586                TYPE = 'FIELD'
     587                if params.get('filename'):
     588                    TYPE = 'FILE'
     589
     590            outdict[name] = value, params
     591
     592        if TYPE == 'RAW':
     593            stream.unget(chunk)
     594
     595        return (TYPE, outdict, stream)
     596
     597
     598class Parser(object):
     599    def __init__(self, stream, boundary):
     600        self._stream = stream
     601        self._separator = '--' + boundary
     602
     603    def __iter__(self):
     604
     605        boundarystream = InterBoundaryIter(self._stream,
     606                                           self._separator)
     607
     608        for sub_stream in boundarystream:
     609            # Iterate over each part
     610            yield ParseBoundaryStream(sub_stream, 1024)
     611
     612def parse_header(line):
     613    """ Parse the header into a key-value. """
     614    plist = _parse_header_params(';' + line)
     615    key = plist.pop(0).lower()
     616    pdict = {}
     617    for p in plist:
     618        i = p.find('=')
     619        if i >= 0:
     620            name = p[:i].strip().lower()
     621            value = p[i+1:].strip()
     622            if len(value) >= 2 and value[0] == value[-1] == '"':
     623                value = value[1:-1]
     624                value = value.replace('\\\\', '\\').replace('\\"', '"')
     625            pdict[name] = value
     626    return key, pdict
     627
     628def _parse_header_params(s):
     629    plist = []
     630    while s[:1] == ';':
     631        s = s[1:]
     632        end = s.find(';')
     633        while end > 0 and s.count('"', 0, end) % 2:
     634            end = s.find(';', end + 1)
     635        if end < 0:
     636            end = len(s)
     637        f = s[:end]
     638        plist.append(f.strip())
     639        s = s[end:]
     640    return plist
  • django/http/__init__.py

     
    99except ImportError:
    1010    from cgi import parse_qsl
    1111
    12 from django.utils.datastructures import MultiValueDict, FileDict
     12from django.utils.datastructures import MultiValueDict, ImmutableList
    1313from django.utils.encoding import smart_str, iri_to_uri, force_unicode
    14 
     14from django.http.multipartparser import MultiPartParser
    1515from utils import *
    1616
    1717RESERVED_CHARS="!*'();:@&=+$,/?%#[]"
     
    2525
    2626    # The encoding used in GET/POST dicts. None means use default setting.
    2727    _encoding = None
     28    _upload_handlers = ()
    2829
    2930    def __init__(self):
    3031        self.GET, self.POST, self.COOKIES, self.META, self.FILES = {}, {}, {}, {}, {}
     
    102103
    103104    encoding = property(_get_encoding, _set_encoding)
    104105
    105 def parse_file_upload(header_dict, post_data):
    106     """Returns a tuple of (POST QueryDict, FILES MultiValueDict)."""
    107     import email, email.Message
    108     from cgi import parse_header
    109     raw_message = '\r\n'.join(['%s:%s' % pair for pair in header_dict.items()])
    110     raw_message += '\r\n\r\n' + post_data
    111     msg = email.message_from_string(raw_message)
    112     POST = QueryDict('', mutable=True)
    113     FILES = MultiValueDict()
    114     for submessage in msg.get_payload():
    115         if submessage and isinstance(submessage, email.Message.Message):
    116             name_dict = parse_header(submessage['Content-Disposition'])[1]
    117             # name_dict is something like {'name': 'file', 'filename': 'test.txt'} for file uploads
    118             # or {'name': 'blah'} for POST fields
    119             # We assume all uploaded files have a 'filename' set.
    120             if 'filename' in name_dict:
    121                 assert type([]) != type(submessage.get_payload()), "Nested MIME messages are not supported"
    122                 if not name_dict['filename'].strip():
    123                     continue
    124                 # IE submits the full path, so trim everything but the basename.
    125                 # (We can't use os.path.basename because that uses the server's
    126                 # directory separator, which may not be the same as the
    127                 # client's one.)
    128                 filename = name_dict['filename'][name_dict['filename'].rfind("\\")+1:]
    129                 FILES.appendlist(name_dict['name'], FileDict({
    130                     'filename': filename,
    131                     'content-type': 'Content-Type' in submessage and submessage['Content-Type'] or None,
    132                     'content': submessage.get_payload(),
    133                 }))
    134             else:
    135                 POST.appendlist(name_dict['name'], submessage.get_payload())
    136     return POST, FILES
     106    def _initialize_handlers(self):
     107        from django.conf import settings
     108        from django.core.files.fileuploadhandler import load_handler
     109        handlers = []
     110        # We go through each handler in the settings variable
     111        # and instantiate the handler by calling HandlerClass(request).
     112        for handler in settings.FILE_UPLOAD_HANDLERS:
     113            handlers.append(load_handler(handler, self))
     114        self._upload_handlers = handlers
    137115
     116    def _set_upload_handlers(self, upload_handlers):
     117        """
     118        Set the upload handler to the new handler given in the parameter.
     119        """
     120        if hasattr(self, '_files'):
     121            raise AttributeError("You cannot set the upload handlers after the upload has been processed.")
     122        self._upload_handlers = upload_handlers
    138123
     124    def _get_upload_handlers(self):
     125        if not self._upload_handlers:
     126            # If thre are no upload handlers defined, initialize them from settings.
     127            self._initialize_handlers()
     128        return self._upload_handlers
     129
     130    upload_handlers = property(_get_upload_handlers, _set_upload_handlers)
     131
     132    def parse_file_upload(self, META, post_data):
     133        """Returns a tuple of (POST QueryDict, FILES MultiValueDict)."""
     134        self.upload_handlers = ImmutableList(self.upload_handlers,
     135                                             warning="You cannot alter the upload handlers after the upload has been processed.")
     136        parser = MultiPartParser(META, post_data, self.upload_handlers,
     137                                 self.encoding)
     138        return parser.parse()
     139
     140
    139141class QueryDict(MultiValueDict):
    140142    """
    141143    A specialized MultiValueDict that takes a query string when initialized.
  • django/test/client.py

     
    1818BOUNDARY = 'BoUnDaRyStRiNg'
    1919MULTIPART_CONTENT = 'multipart/form-data; boundary=%s' % BOUNDARY
    2020
     21
     22class FakePayload(object):
     23    """
     24    A wrapper around StringIO that restricts what can be read,
     25    since data from the network can't be seeked and cannot
     26    be read outside of its content length (or else we hang).
     27    """
     28    def __init__(self, content):
     29        self.__content = StringIO(content)
     30        self.__len = len(content)
     31
     32    def read(self, num_bytes=None):
     33        if num_bytes is None:
     34            num_bytes = self.__len or 1
     35        assert self.__len >= num_bytes, "Cannot read more than the available bytes from the HTTP incoming data."
     36        content = self.__content.read(num_bytes)
     37        self.__len -= num_bytes
     38        return content
     39
     40
    2141class ClientHandler(BaseHandler):
    2242    """
    2343    A HTTP Handler that can be used for testing purposes.
     
    230250            'CONTENT_TYPE':   content_type,
    231251            'PATH_INFO':      urllib.unquote(path),
    232252            'REQUEST_METHOD': 'POST',
    233             'wsgi.input':     StringIO(post_data),
     253            'wsgi.input':     FakePayload(post_data),
    234254        }
    235255        r.update(extra)
    236256
  • django/conf/global_settings.py

     
    224224# Example: "http://media.lawrence.com"
    225225MEDIA_URL = ''
    226226
     227# A tuple that enumerates the upload handlers
     228# in order.
     229FILE_UPLOAD_HANDLERS = (
     230    'django.core.files.fileuploadhandler.MemoryFileUploadHandler',
     231    'django.core.files.fileuploadhandler.TemporaryFileUploadHandler',
     232)
     233
     234# Number of bytes the length of the request can be before it is
     235# streamed to the file system instead of parsed entirely in memory.
     236FILE_UPLOAD_MAX_MEMORY_SIZE = 2621440
     237
     238# Directory to upload streamed files temporarily.
     239# A value of `None` means that it will use the default temporary
     240# directory for the server's operating system.
     241FILE_UPLOAD_TEMP_DIR = None
     242
    227243# Default formatting for date objects. See all available format strings here:
    228244# http://www.djangoproject.com/documentation/templates/#now
    229245DATE_FORMAT = 'N j, Y'
  • django/db/models/base.py

     
    1313from django.utils.datastructures import SortedDict
    1414from django.utils.functional import curry
    1515from django.utils.encoding import smart_str, force_unicode, smart_unicode
     16from django.core.files.filemove import file_move_safe
    1617from django.conf import settings
    1718from itertools import izip
     19import warnings
    1820import types
    1921import sys
    2022import os
     
    384386    def _get_FIELD_size(self, field):
    385387        return os.path.getsize(self._get_FIELD_filename(field))
    386388
    387     def _save_FIELD_file(self, field, filename, raw_contents, save=True):
     389    def _save_FIELD_file(self, field, filename, raw_field, save=True):
    388390        directory = field.get_directory_name()
    389391        try: # Create the date-based directory if it doesn't exist.
    390392            os.makedirs(os.path.join(settings.MEDIA_ROOT, directory))
    391393        except OSError: # Directory probably already exists.
    392394            pass
     395
     396        # Put the deprecation warning first since there are multiple
     397        # locations where we use the new and old interface.
     398        if isinstance(raw_field, dict):
     399            warnings.warn("The dictionary usage for files is deprecated. Use the new object interface instead.", DeprecationWarning)
     400
     401        if filename is None:
     402            try:
     403                filename = raw_field.file_name
     404            except AttributeError:
     405                filename = raw_field['filename']
     406
    393407        filename = field.get_filename(filename)
    394408
    395409        # If the filename already exists, keep adding an underscore to the name of
     
    406420        setattr(self, field.attname, filename)
    407421
    408422        full_filename = self._get_FIELD_filename(field)
    409         fp = open(full_filename, 'wb')
    410         fp.write(raw_contents)
    411         fp.close()
     423        if hasattr(raw_field, 'temporary_file_path'):
     424            # This file has a file path that we can move.
     425            raw_field.close()
     426            file_move_safe(raw_field.temporary_file_path(), full_filename)
     427        else:
     428            from django.core.files import filelocks
     429            fp = open(full_filename, 'wb')
     430            # exclusive lock
     431            filelocks.lock(fp, filelocks.LOCK_EX)
     432            if hasattr(raw_field, 'chunk'):
     433                # This is a normal uploadedfile that we can stream.
     434                for chunk in raw_field.chunk(65535):
     435                    fp.write(chunk)
     436            else:
     437                # This is an old dictionary, use the old interface.
     438                fp.write(raw_field['content'])
     439            filelocks.unlock(fp)
     440            fp.close()
    412441
     442
    413443        # Save the width and/or height, if applicable.
    414444        if isinstance(field, ImageField) and (field.width_field or field.height_field):
    415445            from django.utils.images import get_image_dimensions
  • django/db/models/fields/__init__.py

     
    785785        setattr(cls, 'get_%s_filename' % self.name, curry(cls._get_FIELD_filename, field=self))
    786786        setattr(cls, 'get_%s_url' % self.name, curry(cls._get_FIELD_url, field=self))
    787787        setattr(cls, 'get_%s_size' % self.name, curry(cls._get_FIELD_size, field=self))
    788         setattr(cls, 'save_%s_file' % self.name, lambda instance, filename, raw_contents, save=True: instance._save_FIELD_file(self, filename, raw_contents, save))
     788        setattr(cls, 'save_%s_file' % self.name, lambda instance, filename, raw_field, save=True: instance._save_FIELD_file(self, filename, raw_field, save))
     789        setattr(cls, 'move_%s_file' % self.name, lambda instance, raw_field, save=True: instance._save_FIELD_file(self, None, raw_field, save))
    789790        dispatcher.connect(self.delete_file, signal=signals.post_delete, sender=cls)
    790791
    791792    def delete_file(self, instance):
     
    808809        if new_data.get(upload_field_name, False):
    809810            func = getattr(new_object, 'save_%s_file' % self.name)
    810811            if rel:
    811                 func(new_data[upload_field_name][0]["filename"], new_data[upload_field_name][0]["content"], save)
     812                file = new_data[upload_field_name][0]
    812813            else:
    813                 func(new_data[upload_field_name]["filename"], new_data[upload_field_name]["content"], save)
     814                file = new_data[upload_field_name]
    814815
     816            try:
     817                file_name = file.file_name
     818            except AttributeError:
     819                file_name = file['filename']
     820            func(file_name, file, save)
     821
    815822    def get_directory_name(self):
    816823        return os.path.normpath(force_unicode(datetime.datetime.now().strftime(smart_str(self.upload_to))))
    817824
     
    823830    def save_form_data(self, instance, data):
    824831        from django.newforms.fields import UploadedFile
    825832        if data and isinstance(data, UploadedFile):
    826             getattr(instance, "save_%s_file" % self.name)(data.filename, data.content, save=False)
     833            getattr(instance, "save_%s_file" % self.name)(data.filename, data.data, save=False)
    827834
    828835    def formfield(self, **kwargs):
    829836        defaults = {'form_class': forms.FileField}
  • django/oldforms/__init__.py

     
    680680        self.field_name, self.is_required = field_name, is_required
    681681        self.validator_list = [self.isNonEmptyFile] + validator_list
    682682
    683     def isNonEmptyFile(self, field_data, all_data):
     683    def isNonEmptyFile(self, new_data, all_data):
     684        if hasattr(new_data, 'upload_errors'):
     685            upload_errors = new_data.upload_errors()
     686            if upload_errors:
     687                raise validators.CriticalValidationError, upload_errors
    684688        try:
    685             content = field_data['content']
    686         except TypeError:
    687             raise validators.CriticalValidationError, ugettext("No file was submitted. Check the encoding type on the form.")
    688         if not content:
     689            file_size = new_data.file_size
     690        except AttributeError:
     691            file_size = len(new_data['content'])
     692        if not file_size:
    689693            raise validators.CriticalValidationError, ugettext("The submitted file is empty.")
    690694
    691695    def render(self, data):
    692696        return mark_safe(u'<input type="file" id="%s" class="v%s" name="%s" />' % \
    693697            (self.get_id(), self.__class__.__name__, self.field_name))
    694698
     699    def prepare(self, new_data):
     700        if hasattr(new_data, 'upload_errors'):
     701            upload_errors = new_data.upload_errors()
     702            new_data[self.field_name] = { '_file_upload_error': upload_errors }
     703
    695704    def html2python(data):
    696705        if data is None:
    697706            raise EmptyValue
  • django/core/handlers/wsgi.py

     
    112112        # Populates self._post and self._files
    113113        if self.method == 'POST':
    114114            if self.environ.get('CONTENT_TYPE', '').startswith('multipart'):
    115                 header_dict = dict([(k, v) for k, v in self.environ.items() if k.startswith('HTTP_')])
    116                 header_dict['Content-Type'] = self.environ.get('CONTENT_TYPE', '')
    117                 self._post, self._files = http.parse_file_upload(header_dict, self.raw_post_data)
     115                self._raw_post_data = ''
     116                self._post, self._files = self.parse_file_upload(self.META, self.environ['wsgi.input'])
    118117            else:
    119118                self._post, self._files = http.QueryDict(self.raw_post_data, encoding=self._encoding), datastructures.MultiValueDict()
    120119        else:
  • django/core/handlers/modpython.py

     
    5353    def _load_post_and_files(self):
    5454        "Populates self._post and self._files"
    5555        if 'content-type' in self._req.headers_in and self._req.headers_in['content-type'].startswith('multipart'):
    56             self._post, self._files = http.parse_file_upload(self._req.headers_in, self.raw_post_data)
     56            self._raw_post_data = ''
     57            self._post, self._files = self.parse_file_upload(self.META, self._req)
    5758        else:
    5859            self._post, self._files = http.QueryDict(self.raw_post_data, encoding=self._encoding), datastructures.MultiValueDict()
    5960
  • django/core/files/filelocks.py

     
     1"""
     2Locking portability based partially on example by
     3Jonathan Feignberg <jdf@pobox.com> in python cookbook.
     4
     5Example Usage::
     6
     7    from django.core.files import filelocks
     8
     9    f = open('./file', 'wb')
     10
     11    filelocks.lock(f, filelocks.LOCK_EX)
     12    f.write('Django')
     13    f.close()
     14"""
     15
     16__all__ = ('LOCK_EX','LOCK_SH','LOCK_NB','lock','unlock')
     17
     18system_type = None
     19
     20try:
     21    import win32con
     22    import win32file
     23    import pywintypes
     24    LOCK_EX = win32con.LOCKFILE_EXCLUSIVE_LOCK
     25    LOCK_SH = 0
     26    LOCK_NB = win32con.LOCKFILE_FAIL_IMMEDIATELY
     27    __overlapped = pywintypes.OVERLAPPED()
     28    system_type = 'nt'
     29except (ImportError, AttributeError):
     30    pass
     31
     32try:
     33    import fcntl
     34    LOCK_EX = fcntl.LOCK_EX
     35    LOCK_SH = fcntl.LOCK_SH
     36    LOCK_NB = fcntl.LOCK_NB
     37    system_type = 'posix'
     38except (ImportError, AttributeError):
     39    pass
     40
     41if system_type == 'nt':
     42    def lock(file, flags):
     43        hfile = win32file._get_osfhandle(file.fileno())
     44        win32file.LockFileEx(hfile, flags, 0, -0x10000, __overlapped)
     45
     46    def unlock(file):
     47        hfile = win32file._get_osfhandle(file.fileno())
     48        win32file.UnlockFileEx(hfile, 0, -0x10000, __overlapped)
     49elif system_type == 'posix':
     50    def lock(file, flags):
     51        fcntl.flock(file.fileno(), flags)
     52
     53    def unlock(file):
     54        fcntl.flock(file.fileno(), fcntl.LOCK_UN)
     55else:
     56    # File locking is not supported.
     57    LOCK_EX = LOCK_SH = LOCK_NB = None
     58       
     59    # Dummy functions that don't do anything.
     60    def lock(file, flags):
     61        pass
     62
     63    def unlock(file):
     64        pass
  • django/core/files/uploadedfile.py

     
     1"""
     2The uploaded file objects for Django.
     3This contains the base UploadedFile and the TemporaryUploadedFile
     4derived class.
     5"""
     6
     7__all__ = ('UploadedFile', 'TemporaryUploadedFile', 'InMemoryUploadedFile')
     8
     9class UploadedFile(object):
     10    """
     11    The UploadedFile object behaves somewhat like a file
     12    object and represents some data that the user submitted
     13    and is stored in some form.
     14    """
     15    DEFAULT_CHUNK_SIZE = 64 * 2**10
     16
     17    def __init__(self):
     18        self.file_size = None
     19        self.file_name = None
     20        self.content_type = None
     21        self.charset = None
     22        pass
     23
     24    def file_size(self):
     25        return self.file_size
     26
     27    def chunk(self, chunk_size=None):
     28        """
     29        Read the file to generate chunks of chunk_size bytes.
     30        """
     31        if not chunk_size:
     32            chunk_size = UploadedFile.DEFAULT_CHUNK_SIZE
     33
     34        if hasattr(self, 'seek'):
     35            self.seek(0)
     36        # Assume the pointer is at zero...
     37        counter = self.file_size()
     38
     39        while counter > 0:
     40            yield self.read(chunk_size)
     41            counter -= chunk_size
     42
     43
     44    def multiple_chunks(self, chunk_size=None):
     45        """
     46        Return True if you can expect multiple chunks, False otherwise.
     47        Note: If a particular file representation is in memory, then
     48              override this to return False.
     49        """
     50        if not chunk_size:
     51            chunk_size = UploadedFile.DEFAULT_CHUNK_SIZE
     52        return self.file_size() < chunk_size
     53       
     54
     55    def read(self, num_bytes=None):
     56        """
     57        Read from the file in whatever representation it has.
     58        """
     59        raise NotImplementedError()
     60
     61    def open(self):
     62        """
     63        Open the file, if one needs to.
     64        """
     65        pass
     66
     67
     68    def close(self):
     69        """
     70        Close the file, if one needs to.
     71        """
     72        pass
     73
     74    def __getitem__(self, key):
     75        """
     76        This maintains backwards compatibility.
     77        """
     78        import warnings
     79        warnings.warn("The dictionary access of uploaded file objects is deprecated. Use the new object interface instead.", DeprecationWarning)
     80        # Dictionary to translate labels
     81        # for backwards compatbility.
     82        # Should be removed at some point.
     83        backwards_translate = {
     84            'filename': 'file_name',
     85            'content-type': 'content_type',
     86            }
     87
     88        if key == 'content':
     89            return self.read()
     90        else:
     91            return getattr(self, backwards_translate.get(key, key))
     92
     93    def __repr__(self):
     94        """
     95        This representation could be anything and can be overridden.
     96        This is mostly done to make it look somewhat useful.
     97        """
     98        _dict = {
     99            'file_name': self.file_name,
     100            'content_type': self.content_type,
     101            'content': '<omitted>',
     102            }
     103        return repr(_dict)
     104
     105
     106class TemporaryUploadedFile(UploadedFile):
     107    """
     108    Upload a file to a temporary file.
     109    """
     110
     111    def __init__(self, file, file_name, content_type, file_size, charset):
     112        self.file = file
     113        self.file_name = file_name
     114        self.path = file.name
     115        self.content_type = content_type
     116        self.file_size = file_size
     117        self.charset = charset
     118        self.file.seek(0)
     119
     120    def temporary_file_path(self):
     121        """
     122        Return the full path of this file.
     123        """
     124        return self.path
     125
     126    def read(self, *args, **kwargs):
     127        return self.file.read(*args, **kwargs)
     128
     129    def open(self):
     130        """
     131        Assume the person meant to seek.
     132        """
     133        self.seek(0)
     134
     135    def seek(self, *args, **kwargs):
     136        self.file.seek(*args, **kwargs)
     137
     138
     139class InMemoryUploadedFile(UploadedFile):
     140    """
     141    Upload a file into memory.
     142    """
     143    def __init__(self, file, field_name, file_name, content_type, charset):
     144        self.file = file
     145        self.field_name = field_name
     146        self.file_name = file_name
     147        self.content_type = content_type
     148        self.charset = charset
     149        self.file.seek(0)
     150
     151    def seek(self, *args, **kwargs):
     152        self.file.seek(*args, **kwargs)
     153
     154    def open(self):
     155        self.seek(0)
     156
     157    def read(self, *args, **kwargs):
     158        return self.file.read(*args, **kwargs)
     159
     160    def chunk(self, chunk_size=None):
     161        """
     162        Return the entirety of the data regardless.
     163        """
     164        self.file.seek(0)
     165        return self.read()
     166
     167    def multiple_chunks(self, chunk_size=None):
     168        """
     169        Since it's in memory, we'll never have multiple chunks.
     170        """
     171        return False
     172
     173
     174class SimpleUploadedFile(InMemoryUploadedFile):
     175    """
     176    A simple representation of a file, which
     177    just has content, size, and a name.
     178    """
     179    def __init__(self, name, content, content_type='text/plain'):
     180        try:
     181            from cStringIO import StringIO
     182        except ImportError:
     183            from StringIO import StringIO
     184        self.file = StringIO(content or '')
     185        self.file_name = name
     186        self.field_name = None
     187        self.file_size = len(content or '')
     188        self.content_type = content_type
     189        self.charset = None
     190        self.file.seek(0)
     191
  • django/core/files/__init__.py

     
     1
  • django/core/files/fileuploadhandler.py

     
     1""" A fileuploadhandler base and default subclass for handling file uploads.
     2"""
     3import os
     4try:
     5    from cStringIO import StringIO
     6except ImportError:
     7    from StringIO import StringIO
     8
     9from django.utils.encoding import force_unicode
     10from django.utils.datastructures import MultiValueDict
     11from django.core.exceptions import ImproperlyConfigured
     12
     13from django.core.files.uploadedfile import TemporaryUploadedFile, InMemoryUploadedFile
     14
     15__all__ = ('UploadFileException','StopUpload', 'SkipFile',
     16           'FileUploadHandler', 'TemporaryFileUploadHandler',
     17           'MemoryFileUploadHandler', 'load_handler')
     18
     19
     20class UploadFileException(Exception):
     21    """ Any error having to do with Uploading Files. """
     22    pass
     23
     24class StopUpload(UploadFileException):
     25    """ This exception is raised when an upload must abort. """
     26    pass
     27
     28class SkipFile(UploadFileException):
     29    """ This exception is raised when a file needs to be skipped. """
     30    pass
     31
     32def load_handler(path, *args, **kwargs):
     33    """
     34    Given a path to a handler return the instantiation of that handler.
     35    E.g.::
     36        load_handler('django.core.files.fileuploadhandler.TemporaryFileUploadHandler', request)
     37    will return a TemporaryFileUploadHandler object.
     38    """
     39    i = path.rfind('.')
     40    module, attr = path[:i], path[i+1:]
     41    try:
     42        mod = __import__(module, {}, {}, [attr])
     43    except ImportError, e:
     44        raise ImproperlyConfigured, 'Error importing upload handler module %s: "%s"' % (module, e)
     45    except ValueError, e:
     46        raise ImproperlyConfigured, 'Error importing upload handler module Is FILE_UPLOAD_HANDLERS a correctly defined list or tuple?'
     47    try:
     48        cls = getattr(mod, attr)
     49    except AttributeError:
     50        raise ImproperlyConfigured, 'Module "%s" does not define a "%s" upload handler backend' % (module, attr)
     51    return cls(*args, **kwargs)
     52
     53
     54class FileUploadHandler(object):
     55    """ FileUploadHandler will take data and handle file uploads
     56    in a streamed fashion.
     57    """
     58    chunk_size = 64 * 2 ** 10 #: The default chunk size is 64 KB.
     59
     60    def __init__(self, request=None):
     61        " Initialize some local variables. "
     62        self.file_name = None
     63        self.content_type = None
     64        self.content_length = None
     65        self.charset = None
     66        self.request = request
     67
     68    def handle_raw_input(self, input_data, META, content_length, boundary, encoding=None):
     69        """
     70        Handle the raw input from the client.
     71        Parameters:
     72          *input_data* -- An object that supports reading via .read().
     73          *content_length* -- The (integer) value of the Content-Length header from the client.
     74          *boundary* -- The boundary from the Content-Type header. Be sure to prepend two '--'.
     75        """
     76        pass
     77
     78    def new_file(self, field_name, file_name, content_type, content_length, charset=None):
     79        """
     80        Signal that a new file has been started.
     81       
     82        Warning: Do not trust content_length, if you get it at all.
     83        """
     84        self.field_name = field_name
     85        self.file_name = file_name
     86        self.content_type = content_type
     87        self.content_length = content_length
     88        self.charset = charset
     89
     90    def receive_data_chunk(self, raw_data, start, stop):
     91        """
     92        Receive data from the streamed upload parser.
     93        Start and stop are the positions in the file.
     94        This equality should always be true::
     95            len(raw_data) = stop - start
     96        """
     97        raise NotImplementedError()
     98
     99    def file_complete(self, file_size):
     100        """
     101        Signal that a file has completed.
     102        File size corresponds to the actual size accumulated
     103        by all the chunks.
     104
     105        This should return a valid UploadedFile object.
     106        """
     107        raise NotImplementedError()
     108
     109    def upload_complete(self):
     110        """
     111        Signal that the upload is complete.
     112        Do any cleanup that is necessary for this handler.
     113        """
     114        pass
     115
     116
     117
     118class TemporaryFileUploadHandler(FileUploadHandler):
     119    """
     120    Upload the streaming data into a temporary file.
     121    """
     122    def __init__(self, *args, **kwargs):
     123        """ Import settings for later. """
     124        super(TemporaryFileUploadHandler, self).__init__(*args, **kwargs)
     125        global settings
     126        from django.conf import settings
     127
     128    def new_file(self, file_name, *args, **kwargs):
     129        """
     130        Create the file object to append to as data is coming in.
     131        """
     132        super(TemporaryFileUploadHandler, self).new_file(file_name, *args, **kwargs)
     133        self.file = TemporaryFile(settings.FILE_UPLOAD_TEMP_DIR)
     134        self.write = self.file.write
     135
     136    def receive_data_chunk(self, raw_data, start, stop):
     137        """
     138        Once we get the data, we will save it to our file.
     139        """
     140        self.write(raw_data)
     141
     142    def file_complete(self, file_size):
     143        """
     144        Signal that a file has completed.
     145        File size corresponds to the actual size accumulated
     146        by all the chunks.
     147
     148        This should return a valid UploadedFile object.
     149        """
     150        self.file.seek(0)
     151        return TemporaryUploadedFile(self.file, self.file_name,
     152                                     self.content_type, file_size,
     153                                     self.charset)
     154
     155
     156class MemoryFileUploadHandler(FileUploadHandler):
     157    """
     158    The MemoryFileUploadHandler will place the data directly into memory.
     159    """
     160
     161    def handle_raw_input(self, input_data, META, content_length, boundary, encoding=None):
     162        """
     163        Use the content_length to signal whether or not this handler should be in use.
     164        """
     165        global settings
     166        from django.conf import settings
     167
     168        # If the the post is too large, we cannot use the Memory handler.
     169        if content_length > settings.FILE_UPLOAD_MAX_MEMORY_SIZE:
     170            self.activated = False
     171        else:
     172            self.activated = True
     173
     174    def new_file(self, *args, **kwargs):
     175        super(MemoryFileUploadHandler, self).new_file(*args, **kwargs)
     176        if self.activated:
     177            self.file = StringIO()
     178            return "Stop"
     179
     180    def receive_data_chunk(self, raw_data, start, stop):
     181        """
     182        Add the data to the StringIO file.
     183        """
     184        if self.activated:
     185            self.file.write(raw_data)
     186        else:
     187            return raw_data
     188
     189    def file_complete(self, file_size):
     190        """
     191        Return a file object if we're activated.
     192        """
     193        if not self.activated:
     194            return
     195
     196        return InMemoryUploadedFile(self.file, self.field_name, self.file_name,
     197                                    self.content_type, self.charset)
     198
     199
     200class TemporaryFile(object):
     201    """
     202    A temporary file that tries to delete itself when garbage collected.
     203    """
     204    def __init__(self, dir):
     205        import tempfile
     206        if not dir:
     207            dir = tempfile.gettempdir()
     208        try:
     209            (fd, name) = tempfile.mkstemp(suffix='.upload', dir=dir)
     210            self.file = os.fdopen(fd, 'w+b')
     211        except (OSError, IOError):
     212            raise OSError, "Could not create temporary file for uploading, have you set settings.FILE_UPLOAD_TEMP_DIR correctly?"
     213        self.name = name
     214
     215    def __getattr__(self, name):
     216        a = getattr(self.__dict__['file'], name)
     217        if type(a) != type(0):
     218            setattr(self, name, a)
     219        return a
     220
     221    def __del__(self):
     222        try:
     223            os.unlink(self.name)
     224        except OSError:
     225            pass
  • django/core/files/filemove.py

     
     1import os
     2
     3__all__ = ('file_move_safe',)
     4
     5try:
     6    import shutil
     7    file_move = shutil.move
     8except ImportError:
     9    file_move = os.rename
     10
     11def file_move_safe(old_file_name, new_file_name, chunk_size = 1024*64, allow_overwrite=False):
     12    """
     13    Moves a file from one location to another in the safest way possible.
     14   
     15    First, it tries using shutils.move, which is OS-dependent but doesn't
     16    break with change of filesystems. Then it tries os.rename, which will
     17    break if it encounters a change in filesystems. Lastly, it streams
     18    it manually from one file to another in python.
     19
     20    Without ``allow_overwrite``, if the destination file exists, the
     21    file will raise an IOError.
     22    """
     23
     24    from django.core.files import filelocks
     25
     26    if old_file_name == new_file_name:
     27        # No file moving takes place.
     28        return
     29
     30    if not allow_overwrite and os.path.exists(new_file_name):
     31        raise IOError, "Django does not allow overwriting files."
     32
     33    try:
     34        file_move(old_file_name, new_file_name)
     35        return
     36    except OSError: # moving to another filesystem
     37        pass
     38
     39    new_file = open(new_file_name, 'wb')
     40    # exclusive lock
     41    filelocks.lock(new_file, filelocks.LOCK_EX)
     42    old_file = open(old_file_name, 'rb')
     43    current_chunk = None
     44
     45    while current_chunk != '':
     46        current_chunk = old_file.read(chunk_size)
     47        new_file.write(current_chunk)
     48
     49    new_file.close()
     50    old_file.close()
     51
     52    os.remove(old_file_name)
  • django/utils/datastructures.py

     
    332332            except TypeError: # Special-case if current isn't a dict.
    333333                current = {bits[-1]: v}
    334334
    335 class FileDict(dict):
     335class ImmutableList(tuple):
    336336    """
    337     A dictionary used to hold uploaded file contents. The only special feature
    338     here is that repr() of this object won't dump the entire contents of the
    339     file to the output. A handy safeguard for a large file upload.
     337    A tuple-like object that raises useful
     338    errors when it is asked to mutate.
     339    Example::
     340        a = ImmutableList(range(5), warning=AttributeError("You cannot mutate this."))
     341        a[3] = '4'
     342        (Raises the AttributeError)
    340343    """
    341     def __repr__(self):
    342         if 'content' in self:
    343             d = dict(self, content='<omitted>')
    344             return dict.__repr__(d)
    345         return dict.__repr__(self)
     344
     345    def __new__(cls, *args, **kwargs):
     346        if 'warning' in kwargs:
     347            warning = kwargs['warning']
     348            del kwargs['warning']
     349        else:
     350            warning = 'ImmutableList object is immutable.'
     351        self = tuple.__new__(cls, *args, **kwargs)
     352        self.warning = warning
     353        return self
     354
     355    def complain(self, *wargs, **kwargs):
     356        if isinstance(self.warning, Exception):
     357            raise self.warning
     358        else:
     359            raise AttributeError, self.warning
     360
     361    # All list mutation functions become complain.
     362    __delitem__ = __delslice__ = __iadd__ = __imul__ = complain
     363    __setitem__ = __setslice__ = complain
     364    append = extend = insert = pop = remove = complain
     365    sort = reverse = complain
  • django/utils/text.py

     
    33from django.utils.encoding import force_unicode
    44from django.utils.functional import allow_lazy
    55from django.utils.translation import ugettext_lazy
     6from htmlentitydefs import name2codepoint
    67
    78# Capitalizes the first letter of a string.
    89capfirst = lambda x: x and force_unicode(x)[0].upper() + force_unicode(x)[1:]
     
    218219            yield bit
    219220smart_split = allow_lazy(smart_split, unicode)
    220221
     222def _replace_entity(match):
     223     text = match.group(1)
     224     if text[0] == u'#':
     225         text = text[1:]
     226         try:
     227             if text[0] in u'xX':
     228                 c = int(text[1:], 16)
     229             else:
     230                 c = int(text)
     231             return unichr(c)
     232         except ValueError:
     233             return match.group(0)
     234     else:
     235         try:
     236             return unichr(name2codepoint[text])
     237         except (ValueError, KeyError):
     238             return match.group(0)
     239
     240_entity_re = re.compile(r"&(#?[xX]?(?:[0-9a-fA-F]+|\w{1,8}));")
     241
     242def unescape_entities(text):
     243     return _entity_re.sub(_replace_entity, text)
     244unescape_entities = allow_lazy(unescape_entities, unicode)
  • django/newforms/fields.py

     
    44
    55import copy
    66import datetime
     7import warnings
    78import os
    89import re
    910import time
     
    416417
    417418class UploadedFile(StrAndUnicode):
    418419    "A wrapper for files uploaded in a FileField"
    419     def __init__(self, filename, content):
     420    def __init__(self, filename, data):
    420421        self.filename = filename
    421         self.content = content
     422        self.data = data
    422423
    423424    def __unicode__(self):
    424425        """
     
    444445            return None
    445446        elif not data and initial:
    446447            return initial
     448
     449        if isinstance(data, dict):
     450            # We warn once, then support both ways below.
     451            warnings.warn("The dictionary usage for files is deprecated. Use the new object interface instead.", DeprecationWarning)
     452
    447453        try:
    448             f = UploadedFile(data['filename'], data['content'])
    449         except TypeError:
     454            file_name = data.file_name
     455            file_size = data.file_size
     456        except AttributeError:
     457            try:
     458                file_name = data.get('filename')
     459                file_size = bool(data['content'])
     460            except (AttributeError, KeyError):
     461                raise ValidationError(self.error_messages['invalid'])
     462
     463        if not file_name:
    450464            raise ValidationError(self.error_messages['invalid'])
    451         except KeyError:
    452             raise ValidationError(self.error_messages['missing'])
    453         if not f.content:
     465        if not file_size:
    454466            raise ValidationError(self.error_messages['empty'])
    455         return f
    456467
     468        return UploadedFile(file_name, data)
     469
    457470class ImageField(FileField):
    458471    default_error_messages = {
    459472        'invalid_image': _(u"Upload a valid image. The file you uploaded was either not an image or a corrupted image."),
     
    470483        elif not data and initial:
    471484            return initial
    472485        from PIL import Image
    473         from cStringIO import StringIO
     486
     487        # We need to get the file, it either has a path
     488        # or we have to read it all into memory...
     489        if hasattr(data, 'temporary_file_path'):
     490            file = data.temporary_file_path()
     491        else:
     492            try:
     493                from cStringIO import StringIO
     494            except ImportError:
     495                from StringIO import StringIO
     496            if hasattr(data, 'read'):
     497                file = StringIO(data.read())
     498            else:
     499                file = StringIO(data['content'])
    474500        try:
    475501            # load() is the only method that can spot a truncated JPEG,
    476502            #  but it cannot be called sanely after verify()
    477             trial_image = Image.open(StringIO(f.content))
     503            trial_image = Image.open(file)
    478504            trial_image.load()
    479505            # verify() is the only method that can spot a corrupt PNG,
    480506            #  but it must be called immediately after the constructor
    481             trial_image = Image.open(StringIO(f.content))
     507            trial_image = Image.open(file)
    482508            trial_image.verify()
    483509        except Exception: # Python Imaging Library doesn't recognize it as an image
    484510            raise ValidationError(self.error_messages['invalid_image'])
  • tests/modeltests/model_forms/models.py

     
    7575__test__ = {'API_TESTS': """
    7676>>> from django import newforms as forms
    7777>>> from django.newforms.models import ModelForm
     78>>> from django.core.files.uploadedfile import SimpleUploadedFile
     79>>> from warnings import filterwarnings
     80>>> filterwarnings("ignore")
    7881
    7982The bare bones, absolutely nothing custom, basic case.
    8083
     
    792795
    793796# Upload a file and ensure it all works as expected.
    794797
     798>>> f = TextFileForm(data={'description': u'Assistance'}, files={'file': SimpleUploadedFile('test1.txt', 'hello world')})
     799>>> f.is_valid()
     800True
     801>>> type(f.cleaned_data['file'])
     802<class 'django.newforms.fields.UploadedFile'>
     803>>> instance = f.save()
     804>>> instance.file
     805u'...test1.txt'
     806
     807>>> os.unlink(instance.get_file_filename())
     808
    795809>>> f = TextFileForm(data={'description': u'Assistance'}, files={'file': {'filename': 'test1.txt', 'content': 'hello world'}})
    796810>>> f.is_valid()
    797811True
     
    814828u'...test1.txt'
    815829
    816830# Delete the current file since this is not done by Django.
    817 
    818831>>> os.unlink(instance.get_file_filename())
    819832
    820833# Override the file by uploading a new one.
    821834
    822 >>> f = TextFileForm(data={'description': u'Assistance'}, files={'file': {'filename': 'test2.txt', 'content': 'hello world'}}, instance=instance)
     835>>> f = TextFileForm(data={'description': u'Assistance'}, files={'file': SimpleUploadedFile('test2.txt', 'hello world')}, instance=instance)
    823836>>> f.is_valid()
    824837True
    825838>>> instance = f.save()
    826839>>> instance.file
    827840u'...test2.txt'
    828841
     842# Delete the current file since this is not done by Django.
     843>>> os.unlink(instance.get_file_filename())
     844
     845>>> f = TextFileForm(data={'description': u'Assistance'}, files={'file': {'filename': 'test2.txt', 'content': 'hello world'}})
     846>>> f.is_valid()
     847True
     848>>> instance = f.save()
     849>>> instance.file
     850u'...test2.txt'
     851
     852# Delete the current file since this is not done by Django.
     853>>> os.unlink(instance.get_file_filename())
     854
    829855>>> instance.delete()
    830856
    831857# Test the non-required FileField
     
    838864>>> instance.file
    839865''
    840866
    841 >>> f = TextFileForm(data={'description': u'Assistance'}, files={'file': {'filename': 'test3.txt', 'content': 'hello world'}}, instance=instance)
     867>>> f = TextFileForm(data={'description': u'Assistance'}, files={'file': SimpleUploadedFile('test3.txt', 'hello world')}, instance=instance)
    842868>>> f.is_valid()
    843869True
    844870>>> instance = f.save()
    845871>>> instance.file
    846872u'...test3.txt'
     873
     874# Delete the current file since this is not done by Django.
     875>>> os.unlink(instance.get_file_filename())
    847876>>> instance.delete()
    848877
     878>>> f = TextFileForm(data={'description': u'Assistance'}, files={'file': {'filename': 'test3.txt', 'content': 'hello world'}})
     879>>> f.is_valid()
     880True
     881>>> instance = f.save()
     882>>> instance.file
     883u'...test3.txt'
     884
     885# Delete the current file since this is not done by Django.
     886>>> os.unlink(instance.get_file_filename())
     887>>> instance.delete()
     888
    849889# ImageField ###################################################################
    850890
    851891# ImageField and FileField are nearly identical, but they differ slighty when
     
    858898
    859899>>> image_data = open(os.path.join(os.path.dirname(__file__), "test.png")).read()
    860900
     901>>> f = ImageFileForm(data={'description': u'An image'}, files={'image': SimpleUploadedFile('test.png', image_data)})
     902>>> f.is_valid()
     903True
     904>>> type(f.cleaned_data['image'])
     905<class 'django.newforms.fields.UploadedFile'>
     906>>> instance = f.save()
     907>>> instance.image
     908u'...test.png'
     909
     910# Delete the current file since this is not done by Django.
     911>>> os.unlink(instance.get_image_filename())
     912
    861913>>> f = ImageFileForm(data={'description': u'An image'}, files={'image': {'filename': 'test.png', 'content': image_data}})
    862914>>> f.is_valid()
    863915True
     
    885937
    886938# Override the file by uploading a new one.
    887939
    888 >>> f = ImageFileForm(data={'description': u'Changed it'}, files={'image': {'filename': 'test2.png', 'content': image_data}}, instance=instance)
     940>>> f = ImageFileForm(data={'description': u'Changed it'}, files={'image': SimpleUploadedFile('test2.png', image_data)}, instance=instance)
    889941>>> f.is_valid()
    890942True
    891943>>> instance = f.save()
    892944>>> instance.image
    893945u'...test2.png'
    894946
     947# Delete the current file since this is not done by Django.
     948>>> os.unlink(instance.get_image_filename())
    895949>>> instance.delete()
    896950
     951>>> f = ImageFileForm(data={'description': u'Changed it'}, files={'image': {'filename': 'test2.png', 'content': image_data}})
     952>>> f.is_valid()
     953True
     954>>> instance = f.save()
     955>>> instance.image
     956u'...test2.png'
     957
     958# Delete the current file since this is not done by Django.
     959>>> os.unlink(instance.get_image_filename())
     960>>> instance.delete()
     961
    897962# Test the non-required ImageField
    898963
    899964>>> f = ImageFileForm(data={'description': u'Test'})
     
    904969>>> instance.image
    905970''
    906971
    907 >>> f = ImageFileForm(data={'description': u'And a final one'}, files={'image': {'filename': 'test3.png', 'content': image_data}}, instance=instance)
     972>>> f = ImageFileForm(data={'description': u'And a final one'}, files={'image': SimpleUploadedFile('test3.png', image_data)}, instance=instance)
    908973>>> f.is_valid()
    909974True
    910975>>> instance = f.save()
    911976>>> instance.image
    912977u'...test3.png'
     978
     979# Delete the current file since this is not done by Django.
     980>>> os.unlink(instance.get_image_filename())
    913981>>> instance.delete()
    914982
     983>>> f = ImageFileForm(data={'description': u'And a final one'}, files={'image': {'filename': 'test3.png', 'content': image_data}})
     984>>> f.is_valid()
     985True
     986>>> instance = f.save()
     987>>> instance.image
     988u'...test3.png'
     989>>> instance.delete()
     990
    915991"""}
  • tests/regressiontests/bug639/tests.py

     
    99from regressiontests.bug639.models import Photo
    1010from django.http import QueryDict
    1111from django.utils.datastructures import MultiValueDict
     12from django.core.files.uploadedfile import SimpleUploadedFile
    1213
    1314class Bug639Test(unittest.TestCase):
    1415       
     
    2122       
    2223        # Fake a request query dict with the file
    2324        qd = QueryDict("title=Testing&image=", mutable=True)
    24         qd["image_file"] = {
    25             "filename" : "test.jpg",
    26             "content-type" : "image/jpeg",
    27             "content" : img
    28         }
    29        
     25        #qd["image_file"] = {
     26        #    "filename" : "test.jpg",
     27        #    "content-type" : "image/jpeg",
     28        #    "content" : img
     29        #    }
     30        qd["image_file"] = SimpleUploadedFile('test.jpg', img, 'image/jpeg')
     31
    3032        manip = Photo.AddManipulator()
    3133        manip.do_html2python(qd)
    3234        p = manip.save(qd)
     
    3941        Make sure to delete the "uploaded" file to avoid clogging /tmp.
    4042        """
    4143        p = Photo.objects.get()
    42         os.unlink(p.get_image_filename())
    43  No newline at end of file
     44        os.unlink(p.get_image_filename())
  • tests/regressiontests/forms/error_messages.py

     
    11# -*- coding: utf-8 -*-
    22tests = r"""
    33>>> from django.newforms import *
     4>>> from django.core.files.uploadedfile import SimpleUploadedFile
    45
    56# CharField ###################################################################
    67
     
    214215Traceback (most recent call last):
    215216...
    216217ValidationError: [u'INVALID']
    217 >>> f.clean({})
     218>>> f.clean(SimpleUploadedFile('name', None))
    218219Traceback (most recent call last):
    219220...
    220 ValidationError: [u'MISSING']
    221 >>> f.clean({'filename': 'name', 'content':''})
     221ValidationError: [u'EMPTY FILE']
     222>>> f.clean(SimpleUploadedFile('name', ''))
    222223Traceback (most recent call last):
    223224...
    224225ValidationError: [u'EMPTY FILE']
  • tests/regressiontests/forms/tests.py

     
    2626from regressions import tests as regression_tests
    2727from util import tests as util_tests
    2828from widgets import tests as widgets_tests
     29from warnings import filterwarnings
     30filterwarnings("ignore")
    2931
    3032__test__ = {
    3133    'extra_tests': extra_tests,
  • tests/regressiontests/forms/fields.py

     
    22tests = r"""
    33>>> from django.newforms import *
    44>>> from django.newforms.widgets import RadioFieldRenderer
     5>>> from django.core.files.uploadedfile import SimpleUploadedFile
    56>>> import datetime
    67>>> import time
    78>>> import re
     
    773774>>> f.clean({})
    774775Traceback (most recent call last):
    775776...
    776 ValidationError: [u'No file was submitted.']
     777ValidationError: [u'No file was submitted. Check the encoding type on the form.']
    777778
    778779>>> f.clean({}, '')
    779780Traceback (most recent call last):
    780781...
    781 ValidationError: [u'No file was submitted.']
     782ValidationError: [u'No file was submitted. Check the encoding type on the form.']
    782783
    783784>>> f.clean({}, 'files/test3.pdf')
    784785'files/test3.pdf'
     
    788789...
    789790ValidationError: [u'No file was submitted. Check the encoding type on the form.']
    790791
    791 >>> f.clean({'filename': 'name', 'content': None})
     792>>> f.clean(SimpleUploadedFile('name', None))
    792793Traceback (most recent call last):
    793794...
    794795ValidationError: [u'The submitted file is empty.']
    795796
    796 >>> f.clean({'filename': 'name', 'content': ''})
     797>>> f.clean(SimpleUploadedFile('name', ''))
    797798Traceback (most recent call last):
    798799...
    799800ValidationError: [u'The submitted file is empty.']
    800801
    801 >>> type(f.clean({'filename': 'name', 'content': 'Some File Content'}))
     802>>> type(f.clean(SimpleUploadedFile('name', 'Some File Content')))
    802803<class 'django.newforms.fields.UploadedFile'>
    803804
    804 >>> type(f.clean({'filename': 'name', 'content': 'Some File Content'}, 'files/test4.pdf'))
     805>>> type(f.clean(SimpleUploadedFile('name', 'Some File Content'), 'files/test4.pdf'))
    805806<class 'django.newforms.fields.UploadedFile'>
    806807
    807808# URLField ##################################################################
  • tests/regressiontests/forms/forms.py

     
    11# -*- coding: utf-8 -*-
    22tests = r"""
    33>>> from django.newforms import *
     4>>> from django.core.files.uploadedfile import SimpleUploadedFile
    45>>> import datetime
    56>>> import time
    67>>> import re
     
    14651466>>> print f
    14661467<tr><th>File1:</th><td><ul class="errorlist"><li>This field is required.</li></ul><input type="file" name="file1" /></td></tr>
    14671468
    1468 >>> f = FileForm(data={}, files={'file1': {'filename': 'name', 'content':''}}, auto_id=False)
     1469>>> f = FileForm(data={}, files={'file1': SimpleUploadedFile('name', '')}, auto_id=False)
    14691470>>> print f
    14701471<tr><th>File1:</th><td><ul class="errorlist"><li>The submitted file is empty.</li></ul><input type="file" name="file1" /></td></tr>
    14711472
     
    14731474>>> print f
    14741475<tr><th>File1:</th><td><ul class="errorlist"><li>No file was submitted. Check the encoding type on the form.</li></ul><input type="file" name="file1" /></td></tr>
    14751476
    1476 >>> f = FileForm(data={}, files={'file1': {'filename': 'name', 'content':'some content'}}, auto_id=False)
     1477>>> f = FileForm(data={}, files={'file1': SimpleUploadedFile('name', 'some content')}, auto_id=False)
    14771478>>> print f
    14781479<tr><th>File1:</th><td><input type="file" name="file1" /></td></tr>
    14791480>>> f.is_valid()
  • tests/regressiontests/test_client_regress/views.py

     
    11from django.contrib.auth.decorators import login_required
    22from django.http import HttpResponse, HttpResponseRedirect, HttpResponseServerError
     3import sha
    34
    45def no_template_view(request):
    56    "A simple view that expects a GET request, and returns a rendered template"
     
    1011    Check that a file upload can be updated into the POST dictionary without
    1112    going pear-shaped.
    1213    """
     14    from django.core.files.uploadedfile import UploadedFile
    1315    form_data = request.POST.copy()
    1416    form_data.update(request.FILES)
    15     if isinstance(form_data['file_field'], dict) and isinstance(form_data['name'], unicode):
     17    if isinstance(form_data.get('file_field'), UploadedFile) and isinstance(form_data['name'], unicode):
    1618        return HttpResponse('')
    1719    else:
    1820        return HttpResponseServerError()
    1921
     22def file_upload_view_verify(request):
     23    """
     24    Use the sha digest hash to verify the uploaded contents.
     25    """
     26    from django.core.files.uploadedfile import UploadedFile
     27    form_data = request.POST.copy()
     28    form_data.update(request.FILES)
     29
     30    # Check to see if unicode names worked out.
     31    if not request.FILES['file_unicode'].file_name.endswith(u'test_\u4e2d\u6587_Orl\xe9ans.jpg'):
     32        return HttpResponseServerError()
     33
     34    for key, value in form_data.items():
     35        if key.endswith('_hash'):
     36            continue
     37        if key + '_hash' not in form_data:
     38            continue
     39        submitted_hash = form_data[key + '_hash']
     40        if isinstance(value, UploadedFile):
     41            new_hash = sha.new(value.read()).hexdigest()
     42        else:
     43            new_hash = sha.new(value).hexdigest()
     44        if new_hash != submitted_hash:
     45            return HttpResponseServerError()
     46
     47    return HttpResponse('')
     48
    2049def get_view(request):
    2150    "A simple login protected view"
    2251    return HttpResponse("Hello world")
     
    3766def login_protected_redirect_view(request):
    3867    "A view that redirects all requests to the GET view"
    3968    return HttpResponseRedirect('/test_client_regress/get_view/')
    40 login_protected_redirect_view = login_required(login_protected_redirect_view)
    41  No newline at end of file
     69login_protected_redirect_view = login_required(login_protected_redirect_view)
  • tests/regressiontests/test_client_regress/models.py

     
    55from django.test import Client, TestCase
    66from django.core.urlresolvers import reverse
    77import os
     8import sha
    89
    910class AssertContainsTests(TestCase):
    1011    def test_contains(self):
     
    243244        response = self.client.post('/test_client_regress/file_upload/', post_data)
    244245        self.assertEqual(response.status_code, 200)
    245246
     247    def test_large_upload(self):
     248        import tempfile
     249        dir = tempfile.gettempdir()
     250
     251        (fd, name1) = tempfile.mkstemp(suffix='.file1', dir=dir)
     252        file1 = os.fdopen(fd, 'w+b')
     253        file1.write('a' * (2 ** 21))
     254        file1.seek(0)
     255
     256        (fd, name2) = tempfile.mkstemp(suffix='.file2', dir=dir)
     257        file2 = os.fdopen(fd, 'w+b')
     258        file2.write('a' * (10 * 2 ** 20))
     259        file2.seek(0)
     260
     261        # This file contains chinese symbols for a name.
     262        name3 = os.path.join(dir, u'test_&#20013;&#25991;_Orl\u00e9ans.jpg')
     263        file3 = open(name3, 'w+b')
     264        file3.write('b' * (2 ** 10))
     265        file3.seek(0)
     266
     267        post_data = {
     268            'name': 'Ringo',
     269            'file_field1': file1,
     270            'file_field2': file2,
     271            'file_unicode': file3,
     272            }
     273
     274        for key in post_data.keys():
     275            try:
     276                post_data[key + '_hash'] = sha.new(post_data[key].read()).hexdigest()
     277                post_data[key].seek(0)
     278            except AttributeError:
     279                post_data[key + '_hash'] = sha.new(post_data[key]).hexdigest()
     280
     281        response = self.client.post('/test_client_regress/file_upload_verify/', post_data)
     282
     283        for name in (name1, name2, name3):
     284            try:
     285                os.unlink(name)
     286            except:
     287                pass
     288
     289        self.assertEqual(response.status_code, 200)
     290
     291
    246292class LoginTests(TestCase):
    247293    fixtures = ['testdata']
    248294
  • tests/regressiontests/test_client_regress/urls.py

     
    44urlpatterns = patterns('',
    55    (r'^no_template_view/$', views.no_template_view),
    66    (r'^file_upload/$', views.file_upload_view),
     7    (r'^file_upload_verify/$', views.file_upload_view_verify),
    78    (r'^get_view/$', views.get_view),
    89    url(r'^arg_view/(?P<name>.+)/$', views.view_with_argument, name='arg_view'),
    910    (r'^login_protected_redirect_view/$', views.login_protected_redirect_view)
  • tests/regressiontests/datastructures/tests.py

     
    117117>>> d['person']['2']['firstname']
    118118['Adrian']
    119119
    120 ### FileDict ################################################################
    121 
    122 >>> d = FileDict({'content': 'once upon a time...'})
     120### ImmutableList ################################################################
     121>>> d = ImmutableList(range(10))
     122>>> d.sort()
     123Traceback (most recent call last):
     124  File "<stdin>", line 1, in <module>
     125  File "/var/lib/python-support/python2.5/django/utils/datastructures.py", line 359, in complain
     126    raise AttributeError, self.warning
     127AttributeError: ImmutableList object is immutable.
    123128>>> repr(d)
    124 "{'content': '<omitted>'}"
    125 >>> d = FileDict({'other-key': 'once upon a time...'})
    126 >>> repr(d)
    127 "{'other-key': 'once upon a time...'}"
     129'(0, 1, 2, 3, 4, 5, 6, 7, 8, 9)'
     130>>> d = ImmutableList(range(10), warning="Object is immutable!")
     131>>> d[1]
     1321
     133>>> d[1] = 'test'
     134Traceback (most recent call last):
     135  File "<stdin>", line 1, in <module>
     136  File "/var/lib/python-support/python2.5/django/utils/datastructures.py", line 359, in complain
     137    raise AttributeError, self.warning
     138AttributeError: Object is immutable!
    128139"""
  • AUTHORS

     
    5858    Jökull Sólberg Auðunsson <jokullsolberg@gmail.com>
    5959    Arthur <avandorp@gmail.com>
    6060    David Avsajanishvili <avsd05@gmail.com>
    61     axiak@mit.edu
     61    Mike Axiak <axiak@mit.edu>
    6262    Niran Babalola <niran@niran.org>
    6363    Morten Bagai <m@bagai.com>
    6464    Mikaël Barbero <mikael.barbero nospam at nospam free.fr>
     
    135135    Marc Fargas <telenieko@telenieko.com>
    136136    Szilveszter Farkas <szilveszter.farkas@gmail.com>
    137137    favo@exoweb.net
     138    fdr <drfarina@gmail.com>
    138139    Dmitri Fedortchenko <zeraien@gmail.com>
    139140    Bill Fenner <fenner@gmail.com>
    140141    Stefane Fermgier <sf@fermigier.com>
  • docs/upload_handling.txt

     
     1============
     2File Uploads
     3============
     4
     5**New in Django development version**
     6
     7Most Web sites wouldn't be complete without a way to upload files. Before the
     8file even gets into ``request.FILES`` (discussed in the `Request
     9and response objects`_ documentation) Django has to decide where to put the
     10all the incoming data. This document describes how Django deals with this
     11stage of the upload process.
     12
     13
     14.. _Request and response objects: ../request_response/#attributes
     15
     16Default Behavior
     17================
     18
     19If you set up a quick model that contains a `FileField`_ and use the `Admin
     20interface`_ you can quickly create a form to upload files to your site.
     21By default, if the upload is smaller than *2.5 Megabytes* in size, Django
     22will hold the entire contents of the upload in memory, and the file will
     23quickly be saved to its final destination (as defined by
     24`settings.MEDIA_ROOT`_) without any intermediary.
     25
     26If the entire upload is larger than 2.5 Megabytes, then it will -- by
     27default -- write the contents of the uploaded file to a temporary file
     28in your operating system's default temporary directory. On a posix platform,
     29this means that you could expect a file similar to ``/tmp/tmpzfp6I6.upload``
     30to be generated and filled during the upload process.
     31
     32.. note::
     33    You may find that the default temporary directory is not writable.
     34    This is especially true if you are on shared hosting. If this is
     35    the case, Django will raise an error when you try to upload. Please
     36    read below in `Extending the Default`_ to change this directory.
     37
     38
     39.. _FileField: ../model-api/#filefield
     40.. _Admin interface: ../tutorial02/#activate-the-admin-site
     41.. _settings.MEDIA_ROOT: ../settings/#media-root
     42
     43Extending the Default
     44=====================
     45
     46If you have quite a bit of memory, you may decide that you want Django to
     47stream only if the entire upload exceeds 10 Megabytes. You may even
     48decide that whatever directory is the platform default is not suitable
     49for your project. If this is the case, Django provides these two settings:
     50
     51    =============================== ======================================
     52    Setting                         Description
     53    =============================== ======================================
     54    ``FILE_UPLOAD_MAX_MEMORY_SIZE`` The maximum size of a request in bytes
     55                                    for which Django will try to load the
     56                                    entire upload contents in memory.
     57
     58    ``FILE_UPLOAD_TEMP_DIR``        The directory on the file system where
     59                                    uploaded contents will be temporarily
     60                                    stored if not completely in memory.
     61                                    E.g.: ``"/tmp"``
     62    =============================== ======================================
     63
     64There is one final setting -- ``FILE_UPLOAD_HANDLERS`` -- which allows complete
     65customization of the upload process.
     66
     67Upload Handlers
     68===============
     69
     70Through upload handlers Django provides the flexibility to extend the
     71upload process beyond simple storage. You can use custom handlers to enforce
     72user-level quotas, compress data on the fly, render progress bars, and
     73even send data to another warehouse directly without storing it locally!
     74
     75There are two pieces to the Django upload handling: the upload handler and the
     76uploaded file. These are both represented by python classes --
     77``FileUploadHandler`` and ``UploadedFile`` respectively. Throughout the lifetime
     78of the upload process, Django will call on the upload handler to handle the
     79upload, while the upload handler is expected to return an ``UploadedFile``
     80object at the end of a file's upload.
     81
     82Setting Default Upload Handlers for your Project
     83------------------------------------------------
     84
     85Similar to `Middleware`_, upload handlers have an order that is initially
     86defined in ``settings.FILE_UPLOAD_HANDLERS``. The default value is::
     87
     88    ("django.core.files.fileuploadhandler.MemoryFileUploadHandler",
     89     "django.core.files.fileuploadhandler.TemporaryFileUploadHandler",)
     90
     91This literally means: Try putting the upload in memory first and failing
     92that put the upload in a temporary file. However, this behavior is
     93completely dependent on how each of those handlers is defined. For instance,
     94just because there isn't an example, there's no reason an upload handler
     95could run first recording information.
     96
     97.. _Middleware: ../middleware/
     98
     99Modifying your Upload Handlers Dynamically
     100------------------------------------------
     101
     102During the lifetime of your project, you may realize that a particular
     103view or views require different uploading behavior. For this reason,
     104the ``request`` object contains a list of handlers
     105(``request.upload_handlers``) that will be called in order. To append
     106a handler to the list, you would append to it like any other list.
     107For example, suppose you had an ``ProgressBarUploadHandler`` class.
     108To append it to your upload handlers you would write::
     109
     110    request.upload_handlers.append(ProgressBarUploadHandler())
     111
     112However, since the progress bar handler would probably need to run before the
     113other handlers get a chance, you'd probably want to insert it. That is, you'd
     114write::
     115
     116   request.upload_handlers.insert(0, ProgressBarUploadHandler())
     117
     118If you want to replace the upload handlers completely, you can just assign a new
     119list::
     120
     121   request.upload_handlers = [ProgressBarUploadHandler()]
     122
     123And all Django will do is keep a progress, but do nothing with the file itself!
     124One more note: After the upload has completed, you are no longer allowed to
     125assign or modify this list, and Django will raise an error if you try to modify
     126this list after an upload.
     127
     128Writing a File Upload Handler
     129-----------------------------
     130
     131All file upload handlers are subclasses of ``FileUploadHandler``, found in
     132``django.core.files.fileuploadhandler``. To create your handler, you need to
     133define the required methods followed by the methods that make most sense to you:
     134
     135chunk_size
     136~~~~~~~~~~
     137
     138This is an integer attribute that specifies how large the chunks we should put
     139into memory from the network are. The chunk sizes should be divisible by ``4``
     140and should not exceed ``2 ** 31 - 1`` in size. When there are multiple chunk
     141sizes provided by multiple handlers, Django will use the smallest chunk size.
     142
     143new_file
     144~~~~~~~~
     145
     146This signals that a new file is starting. You can initialize a file or whatever
     147else is needed on a new file upload.
     148
     149Interface: ``new_file(self, field_name, file_name, content_type, content_length,
     150charset)``
     151
     152``field_name`` is a string name of the field this was POSTed as.
     153
     154``file_name`` is the unicode filename that was provided by the browser.
     155
     156``content_type`` is the MIME type provided by the browser -- E.g.
     157``'image/jpeg'``.
     158
     159``content_length`` is the length of the image given by the browser if provided,
     160``None`` otherwise.
     161
     162``charset`` is the charset given by the browser if provided, ``None`` otherwise.
     163
     164Returns: ``None`` if you want other handlers to get ``new_file`` called.
     165Something nonzero if you don't want subsequent handlers to get a chance.
     166
     167receive_data_chunk
     168~~~~~~~~~~~~~~~~~~
     169*required*
     170
     171This method is used to do something with the new chunk of data. For example: The
     172``TemporaryFileUploadHandler`` takes the data and writes it to disk.
     173
     174Interface: ``receive_data_chunk(self, raw_data, start, stop)``
     175
     176``raw_data`` is a byte string containing the uploaded data.
     177
     178``start`` is the byte number where the chunk starts.
     179
     180``stop`` is the byte number where the chunk stops.
     181
     182Returns: ``None`` if you don't want the subsequent upload handlers to receive
     183the data. Whatever else you return gets fed into the subsequent upload handlers'
     184``receive_data_chunk`` method.
     185
     186Exceptions: If you raise a ``StopUpload`` or a ``SkipFile`` exception, the
     187upload will abort or the file will be skipped respectively.
     188
     189file_complete
     190~~~~~~~~~~~~~
     191*required*
     192
     193This method defines when a function has finished uploading and gets packaged
     194into an ``UploadedFile`` object.
     195
     196Interface: ``file_complete(self, file_size)``
     197
     198``file_size`` is the number of bytes you have received for this file.
     199
     200Returns: An ``UploadedFile`` object to set in the ``request.FILES`` dictionary.
     201``None`` if you want the subsequent upload handlers to get an ``UploadedFile``
     202object.
     203
     204upload_complete
     205~~~~~~~~~~~~~~~
     206
     207This method defines when the entire upload has completed.
     208
     209Interface: ``upload_complete(self)``
     210
     211handle_raw_input
     212~~~~~~~~~~~~~~~~
     213
     214This method allows the handler to completely override the parsing of the raw
     215HTTP-layered input.
     216
     217Interface: ``handle_raw_input(self, input_data, META, content_length, boundary,
     218encoding)``
     219
     220``input_data`` is a file-like object that supports the read operation.
     221
     222``META`` is the same object as ``request.META``.
     223
     224``content_length`` is the length of the data in ``input_data``. Don't read more
     225than ``content_length`` bytes from ``input_data``.
     226
     227``boundary`` is the MIME boundary for this request.
     228
     229``encoding`` is the encoding of the request.
     230
     231Returns: ``None`` if you want to go on to the next stage, ``(POST, FILES)`` if
     232you want to return the new data structures suitable for the request directly.
     233
     234Defining an Uploaded File
     235-------------------------
     236
     237All file upload handlers are subclasses of ``UploadedFile``, found in
     238``django.core.files.uploadedfile``. The uploaded file object is returned by the
     239handler above in ``file_complete``. To create your own uploaded file class, you
     240need to define the required methods followed by the methods that make most sense
     241to you:
     242
     243read
     244~~~~
     245*required*
     246
     247Interface: ``read(self, num_bytes=None)``
     248
     249Returns: A byte string of length ``num_bytes`` (or the size of the file if
     250``num_bytes`` is not supplied).
     251
     252chunk
     253~~~~~
     254
     255A generator to yield small chunks from the file. With the ``read()`` defined,
     256the ``UploadedFile`` class already defines a ``chunk()`` method that's probably
     257suitable.
     258
     259Interface: ``chunk(self, chunk_size=None)``
     260
     261multiple_chunks
     262~~~~~~~~~~~~~~~
     263
     264Interface: ``multiple_chunks(self, chunk_size=None)``
     265
     266Returns: ``True`` or ``False`` depending on whether or not the user of this file
     267can expect more than one chunk when calling ``chunk(self, chunk_size)``. If all
     268the data is in memory, you should return ``False``.
     269
     270temporary_file_path
     271~~~~~~~~~~~~~~~~~~~
     272
     273If defined, this method should return the file path of the file on the operating
     274system. This will let users move files rather than write to new files if the
     275option is available.
     276
     277Interface: ``temporary_file_path(self)``
     278
     279Returns: A file path in a file system on the local computer.
     280
     281upload_errors
     282~~~~~~~~~~~~~
     283
     284If defined, this method should return a string describing errors that occured
     285during uploads.
     286
     287Interface: ``upload_errors(self)``
     288
     289Returns: A unicode string describing an error that occured with the file's upload.
  • docs/settings.txt

     
    513513The character encoding used to decode any files read from disk. This includes
    514514template files and initial SQL data files.
    515515
     516FILE_UPLOAD_HANDLERS
     517--------------------
     518
     519**New in Django development version**
     520
     521Default::
     522
     523    ("django.core.files.fileuploadhandler.MemoryFileUploadHandler",
     524     "django.core.files.fileuploadhandler.TemporaryFileUploadHandler",)
     525
     526A tuple of handlers to use for uploading.
     527
     528FILE_UPLOAD_MAX_MEMORY_SIZE
     529---------------------------
     530
     531**New in Django development version**
     532
     533Default: ``2621440``
     534
     535The maximum size (in bytes) that an upload will be before it gets streamed to the file system.
     536
     537FILE_UPLOAD_TEMP_DIR
     538--------------------
     539
     540**New in Django development version**
     541
     542Default: ``None``
     543
     544The directory to store data temporarily while uploading files. If ``None``, Django will use the standard temporary directory for the operating system. For example, this will default to '/tmp' on *nix-style operating systems.
     545
    516546FIXTURE_DIRS
    517547-------------
    518548
  • docs/newforms.txt

     
    805805need to bind the file data containing the mugshot image::
    806806
    807807    # Bound form with an image field
     808    >>> from django.core.files.uploadedfile import SimpleUploadedFile
    808809    >>> data = {'subject': 'hello',
    809810    ...         'message': 'Hi there',
    810811    ...         'sender': 'foo@example.com',
    811812    ...         'cc_myself': True}
    812     >>> file_data = {'mugshot': {'filename':'face.jpg'
    813     ...                          'content': <file data>}}
     813    >>> file_data = {'mugshot': SimpleUploadedFile('face.jpg', <file data>)}
    814814    >>> f = ContactFormWithMugshot(data, file_data)
    815815
    816816In practice, you will usually specify ``request.FILES`` as the source
Back to Top