Ticket #2070: 2070-r7728.patch

File 2070-r7728.patch, 114.5 KB (added by jacob, 7 years ago)

"release candidate" patch

  • AUTHORS

    diff --git a/AUTHORS b/AUTHORS
    index 94dc889..28ac6b7 100644
    a b answer newbie questions, and generally made Django that much better: 
    5959    Arthur <avandorp@gmail.com>
    6060    av0000@mail.ru
    6161    David Avsajanishvili <avsd05@gmail.com>
    62     axiak@mit.edu
     62    Mike Axiak <axiak@mit.edu>
    6363    Niran Babalola <niran@niran.org>
    6464    Morten Bagai <m@bagai.com>
    6565    Mikaël Barbero <mikael.barbero nospam at nospam free.fr>
    answer newbie questions, and generally made Django that much better: 
    140140    Marc Fargas <telenieko@telenieko.com>
    141141    Szilveszter Farkas <szilveszter.farkas@gmail.com>
    142142    favo@exoweb.net
     143    fdr <drfarina@gmail.com>
    143144    Dmitri Fedortchenko <zeraien@gmail.com>
     145    Jonathan Feignberg <jdf@pobox.com>
    144146    Liang Feng <hutuworm@gmail.com>
    145147    Bill Fenner <fenner@gmail.com>
    146148    Stefane Fermgier <sf@fermigier.com>
  • django/conf/global_settings.py

    diff --git a/django/conf/global_settings.py b/django/conf/global_settings.py
    index 006ab42..2c9720d 100644
    a b MEDIA_ROOT = '' 
    231231# Example: "http://media.lawrence.com"
    232232MEDIA_URL = ''
    233233
     234# List of upload handler classes to be applied in order.
     235FILE_UPLOAD_HANDLERS = (
     236    'django.core.files.uploadhandler.MemoryFileUploadHandler',
     237    'django.core.files.uploadhandler.TemporaryFileUploadHandler',
     238)
     239
     240# Maximum size, in bytes, of a request before it will be streamed to the
     241# file system instead of into memory.
     242FILE_UPLOAD_MAX_MEMORY_SIZE = 2621440 # i.e. 2.5 MB
     243
     244# Directory in which upload streamed files will be temporarily saved. A value of
     245# `None` will make Django use the operating system's default temporary directory
     246# (i.e. "/tmp" on *nix systems).
     247FILE_UPLOAD_TEMP_DIR = None
     248
    234249# Default formatting for date objects. See all available format strings here:
    235250# http://www.djangoproject.com/documentation/templates/#now
    236251DATE_FORMAT = 'N j, Y'
  • new file django/core/files/locks.py

    diff --git a/django/core/files/__init__.py b/django/core/files/__init__.py
    new file mode 100644
    index 0000000..e69de29
    diff --git a/django/core/files/locks.py b/django/core/files/locks.py
    new file mode 100644
    index 0000000..212b51a
    - +  
     1"""
     2Portable file locking utilities.
     3
     4Based partially on example by Jonathan Feignberg <jdf@pobox.com> in the Python
     5Cookbook, licensed under the Python Software License.
     6
     7    http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/65203
     8
     9Example Usage::
     10
     11    >>> from django.core.files import locks
     12    >>> f = open('./file', 'wb')
     13    >>> locks.lock(f, locks.LOCK_EX)
     14    >>> f.write('Django')
     15    >>> f.close()
     16"""
     17
     18__all__ = ('LOCK_EX','LOCK_SH','LOCK_NB','lock','unlock')
     19
     20system_type = None
     21
     22try:
     23    import win32con
     24    import win32file
     25    import pywintypes
     26    LOCK_EX = win32con.LOCKFILE_EXCLUSIVE_LOCK
     27    LOCK_SH = 0
     28    LOCK_NB = win32con.LOCKFILE_FAIL_IMMEDIATELY
     29    __overlapped = pywintypes.OVERLAPPED()
     30    system_type = 'nt'
     31except (ImportError, AttributeError):
     32    pass
     33
     34try:
     35    import fcntl
     36    LOCK_EX = fcntl.LOCK_EX
     37    LOCK_SH = fcntl.LOCK_SH
     38    LOCK_NB = fcntl.LOCK_NB
     39    system_type = 'posix'
     40except (ImportError, AttributeError):
     41    pass
     42
     43if system_type == 'nt':
     44    def lock(file, flags):
     45        hfile = win32file._get_osfhandle(file.fileno())
     46        win32file.LockFileEx(hfile, flags, 0, -0x10000, __overlapped)
     47
     48    def unlock(file):
     49        hfile = win32file._get_osfhandle(file.fileno())
     50        win32file.UnlockFileEx(hfile, 0, -0x10000, __overlapped)
     51elif system_type == 'posix':
     52    def lock(file, flags):
     53        fcntl.flock(file.fileno(), flags)
     54
     55    def unlock(file):
     56        fcntl.flock(file.fileno(), fcntl.LOCK_UN)
     57else:
     58    # File locking is not supported.
     59    LOCK_EX = LOCK_SH = LOCK_NB = None
     60
     61    # Dummy functions that don't do anything.
     62    def lock(file, flags):
     63        pass
     64
     65    def unlock(file):
     66        pass
  • new file django/core/files/move.py

    diff --git a/django/core/files/move.py b/django/core/files/move.py
    new file mode 100644
    index 0000000..66873d4
    - +  
     1"""
     2Move a file in the safest way possible::
     3
     4    >>> from django.core.files.move import file_move_save
     5    >>> file_move_save("/tmp/old_file", "/tmp/new_file")
     6"""
     7
     8import os
     9from django.core.files import locks
     10
     11__all__ = ['file_move_safe']
     12
     13try:
     14    import shutil
     15    file_move = shutil.move
     16except ImportError:
     17    file_move = os.rename
     18
     19def file_move_safe(old_file_name, new_file_name, chunk_size = 1024*64, allow_overwrite=False):
     20    """
     21    Moves a file from one location to another in the safest way possible.
     22
     23    First, try using ``shutils.move``, which is OS-dependent but doesn't break
     24    if moving across filesystems. Then, try ``os.rename``, which will break
     25    across filesystems. Finally, streams manually from one file to another in
     26    pure Python.
     27
     28    If the destination file exists and ``allow_overwrite`` is ``False``, this
     29    function will throw an ``IOError``.
     30    """
     31
     32    # There's no reason to move if we don't have to.
     33    if old_file_name == new_file_name:
     34        return
     35
     36    if not allow_overwrite and os.path.exists(new_file_name):
     37        raise IOError("Cannot overwrite existing file '%s'." % new_file_name)
     38
     39    try:
     40        file_move(old_file_name, new_file_name)
     41        return
     42    except OSError:
     43        # This will happen with os.rename if moving to another filesystem
     44        pass
     45
     46    # If the built-in didn't work, do it the hard way.
     47    new_file = open(new_file_name, 'wb')
     48    locks.lock(new_file, locks.LOCK_EX)
     49    old_file = open(old_file_name, 'rb')
     50    current_chunk = None
     51
     52    while current_chunk != '':
     53        current_chunk = old_file.read(chunk_size)
     54        new_file.write(current_chunk)
     55
     56    new_file.close()
     57    old_file.close()
     58
     59    os.remove(old_file_name)
  • new file django/core/files/uploadedfile.py

    diff --git a/django/core/files/uploadedfile.py b/django/core/files/uploadedfile.py
    new file mode 100644
    index 0000000..51cec17
    - +  
     1"""
     2Classes representing uploaded files.
     3"""
     4
     5import os
     6try:
     7    from cStringIO import StringIO
     8except ImportError:
     9    from StringIO import StringIO
     10
     11__all__ = ('UploadedFile', 'TemporaryUploadedFile', 'InMemoryUploadedFile')
     12
     13class UploadedFile(object):
     14    """
     15    A abstract uploadded file (``TemporaryUploadedFile`` and
     16    ``InMemoryUploadedFile`` are the built-in concrete subclasses).
     17
     18    An ``UploadedFile`` object behaves somewhat like a file object and
     19    represents some file data that the user submitted with a form.
     20    """
     21    DEFAULT_CHUNK_SIZE = 64 * 2**10
     22
     23    def __init__(self, file_name=None, content_type=None, file_size=None, charset=None):
     24        self.file_name = file_name
     25        self.file_size = file_size
     26        self.content_type = content_type
     27        self.charset = charset
     28
     29    def __repr__(self):
     30        return "<%s: %s (%s)>" % (self.__class__.__name__, self.file_name, self.content_type)
     31
     32    def _set_file_name(self, name):
     33        # Sanitize the file name so that it can't be dangerous.
     34        if name is not None:
     35            # Just use the basename of the file -- anything else is dangerous.
     36            name = os.path.basename(name)
     37           
     38            # File names longer than 255 characters can cause problems on older OSes.
     39            if len(name) > 255:
     40                name, ext = os.path.splitext(name)
     41                name = name[:255 - len(ext)] + ext
     42               
     43        self._file_name = name
     44       
     45    def _get_file_name(self):
     46        return self._file_name
     47       
     48    file_name = property(_get_file_name, _set_file_name)
     49
     50    def chunk(self, chunk_size=None):
     51        """
     52        Read the file and yield chucks of ``chunk_size`` bytes (defaults to
     53        ``UploadedFile.DEFAULT_CHUNK_SIZE``).
     54        """
     55        if not chunk_size:
     56            chunk_size = UploadedFile.DEFAULT_CHUNK_SIZE
     57
     58        if hasattr(self, 'seek'):
     59            self.seek(0)
     60        # Assume the pointer is at zero...
     61        counter = self.file_size
     62
     63        while counter > 0:
     64            yield self.read(chunk_size)
     65            counter -= chunk_size
     66
     67    def multiple_chunks(self, chunk_size=None):
     68        """
     69        Returns ``True`` if you can expect multiple chunks.
     70
     71        NB: If a particular file representation is in memory, subclasses should
     72        always return ``False`` -- there's no good reason to read from memory in
     73        chunks.
     74        """
     75        if not chunk_size:
     76            chunk_size = UploadedFile.DEFAULT_CHUNK_SIZE
     77        return self.file_size < chunk_size
     78
     79    # Abstract methods; subclasses *must* default read() and probably should
     80    # define open/close.
     81    def read(self, num_bytes=None):
     82        raise NotImplementedError()
     83
     84    def open(self):
     85        pass
     86
     87    def close(self):
     88        pass
     89
     90    # Backwards-compatible support for uploaded-files-as-dictionaries.
     91    def __getitem__(self, key):
     92        import warnings
     93        warnings.warn(
     94            message = "The dictionary access of uploaded file objects is deprecated. Use the new object interface instead.",
     95            category = DeprecationWarning,
     96            stacklevel = 2
     97        )
     98        backwards_translate = {
     99            'filename': 'file_name',
     100            'content-type': 'content_type',
     101            }
     102
     103        if key == 'content':
     104            return self.read()
     105        elif key == 'filename':
     106            return self.file_name
     107        elif key == 'content-type':
     108            return self.content_type
     109        else:
     110            return getattr(self, key)
     111
     112class TemporaryUploadedFile(UploadedFile):
     113    """
     114    A file uploaded to a temporary location (i.e. stream-to-disk).
     115    """
     116
     117    def __init__(self, file, file_name, content_type, file_size, charset):
     118        super(TemporaryUploadedFile, self).__init__(file_name, content_type, file_size, charset)
     119        self.file = file
     120        self.path = file.name
     121        self.file.seek(0)
     122
     123    def temporary_file_path(self):
     124        """
     125        Returns the full path of this file.
     126        """
     127        return self.path
     128
     129    def read(self, *args, **kwargs):
     130        return self.file.read(*args, **kwargs)
     131
     132    def open(self):
     133        self.seek(0)
     134
     135    def seek(self, *args, **kwargs):
     136        self.file.seek(*args, **kwargs)
     137
     138class InMemoryUploadedFile(UploadedFile):
     139    """
     140    A file uploaded into memory (i.e. stream-to-memory).
     141    """
     142    def __init__(self, file, field_name, file_name, content_type, charset, file_size):
     143        super(InMemoryUploadedFile, self).__init__(file_name, content_type, charset, file_size)
     144        self.file = file
     145        self.field_name = field_name
     146        self.file.seek(0)
     147
     148    def seek(self, *args, **kwargs):
     149        self.file.seek(*args, **kwargs)
     150
     151    def open(self):
     152        self.seek(0)
     153
     154    def read(self, *args, **kwargs):
     155        return self.file.read(*args, **kwargs)
     156
     157    def chunk(self, chunk_size=None):
     158        self.file.seek(0)
     159        yield self.read()
     160
     161    def multiple_chunks(self, chunk_size=None):
     162        # Since it's in memory, we'll never have multiple chunks.
     163        return False
     164
     165class SimpleUploadedFile(InMemoryUploadedFile):
     166    """
     167    A simple representation of a file, which just has content, size, and a name.
     168    """
     169    def __init__(self, name, content, content_type='text/plain'):
     170        self.file = StringIO(content or '')
     171        self.file_name = name
     172        self.field_name = None
     173        self.file_size = len(content or '')
     174        self.content_type = content_type
     175        self.charset = None
     176        self.file.seek(0)
     177
     178    def from_dict(cls, file_dict):
     179        """
     180        Creates a SimpleUploadedFile object from
     181        a dictionary object with the following keys:
     182           - filename
     183           - content-type
     184           - content
     185        """
     186        return cls(file_dict['filename'],
     187                   file_dict['content'],
     188                   file_dict.get('content-type', 'text/plain'))
     189
     190    from_dict = classmethod(from_dict)
  • new file django/core/files/uploadhandler.py

    diff --git a/django/core/files/uploadhandler.py b/django/core/files/uploadhandler.py
    new file mode 100644
    index 0000000..0349539
    - +  
     1"""
     2Base file upload handler classes, and the built-in concrete subclasses
     3"""
     4import os
     5import tempfile
     6try:
     7    from cStringIO import StringIO
     8except ImportError:
     9    from StringIO import StringIO
     10
     11from django.conf import settings
     12from django.core.exceptions import ImproperlyConfigured
     13from django.core.files.uploadedfile import TemporaryUploadedFile, InMemoryUploadedFile
     14
     15__all__ = ['UploadFileException','StopUpload', 'SkipFile', 'FileUploadHandler',
     16           'TemporaryFileUploadHandler', 'MemoryFileUploadHandler',
     17           'load_handler']
     18
     19class UploadFileException(Exception):
     20    """
     21    Any error having to do with uploading files.
     22    """
     23    pass
     24
     25class StopUpload(UploadFileException):
     26    """
     27    This exception is raised when an upload must abort.
     28    """
     29    def __init__(self, connection_reset=False):
     30        """
     31        If ``connection_reset`` is ``True``, Django knows will halt the upload
     32        without consuming the rest of the upload. This will cause the browser to
     33        show a "connection reset" error.
     34        """
     35        self.connection_reset = connection_reset
     36
     37    def __unicode__(self):
     38        if self.connection_reset:
     39            return u'StopUpload: Halt current upload.'
     40        else:
     41            return u'StopUpload: Consume request data, then halt.'
     42
     43class SkipFile(UploadFileException):
     44    """
     45    This exception is raised by an upload handler that wants to skip a given file.
     46    """
     47    pass
     48   
     49class StopFutureHandlers(UploadFileException):
     50    """
     51    Upload handers that have handled a file and do not want future handlers to
     52    run should raise this exception instead of returning None.
     53    """
     54    pass
     55
     56class FileUploadHandler(object):
     57    """
     58    Base class for streaming upload handlers.
     59    """
     60    chunk_size = 64 * 2 ** 10 #: The default chunk size is 64 KB.
     61
     62    def __init__(self, request=None):
     63        self.file_name = None
     64        self.content_type = None
     65        self.content_length = None
     66        self.charset = None
     67        self.request = request
     68
     69    def handle_raw_input(self, input_data, META, content_length, boundary, encoding=None):
     70        """
     71        Handle the raw input from the client.
     72
     73        Parameters:
     74
     75            :input_data:
     76                An object that supports reading via .read().
     77            :META:
     78                ``request.META``.
     79            :content_length:
     80                The (integer) value of the Content-Length header from the
     81                client.
     82            :boundary: The boundary from the Content-Type header. Be sure to
     83                prepend two '--'.
     84        """
     85        pass
     86
     87    def new_file(self, field_name, file_name, content_type, content_length, charset=None):
     88        """
     89        Signal that a new file has been started.
     90
     91        Warning: As with any data from the client, you should not trust
     92        content_length (and sometimes won't even get it).
     93        """
     94        self.field_name = field_name
     95        self.file_name = file_name
     96        self.content_type = content_type
     97        self.content_length = content_length
     98        self.charset = charset
     99
     100    def receive_data_chunk(self, raw_data, start):
     101        """
     102        Receive data from the streamed upload parser. ``start`` is the position
     103        in the file of the chunk.
     104        """
     105        raise NotImplementedError()
     106
     107    def file_complete(self, file_size):
     108        """
     109        Signal that a file has completed. File size corresponds to the actual
     110        size accumulated by all the chunks.
     111
     112        Subclasses must should return a valid ``UploadedFile`` object.
     113        """
     114        raise NotImplementedError()
     115
     116    def upload_complete(self):
     117        """
     118        Signal that the upload is complete. Subclasses should perform cleanup
     119        that is necessary for this handler.
     120        """
     121        pass
     122
     123class TemporaryFileUploadHandler(FileUploadHandler):
     124    """
     125    Upload handler that streams data into a temporary file.
     126    """
     127    def __init__(self, *args, **kwargs):
     128        super(TemporaryFileUploadHandler, self).__init__(*args, **kwargs)
     129
     130    def new_file(self, file_name, *args, **kwargs):
     131        """
     132        Create the file object to append to as data is coming in.
     133        """
     134        super(TemporaryFileUploadHandler, self).new_file(file_name, *args, **kwargs)
     135        self.file = TemporaryFile(settings.FILE_UPLOAD_TEMP_DIR)
     136        self.write = self.file.write
     137
     138    def receive_data_chunk(self, raw_data, start):
     139        self.write(raw_data)
     140
     141    def file_complete(self, file_size):
     142        self.file.seek(0)
     143        return TemporaryUploadedFile(self.file, self.file_name,
     144                                     self.content_type, file_size,
     145                                     self.charset)
     146
     147class MemoryFileUploadHandler(FileUploadHandler):
     148    """
     149    File upload handler to stream uploads into memory (used for small files).
     150    """
     151
     152    def handle_raw_input(self, input_data, META, content_length, boundary, encoding=None):
     153        """
     154        Use the content_length to signal whether or not this handler should be in use.
     155        """
     156        # Check the content-length header to see if we should
     157        # If the the post is too large, we cannot use the Memory handler.
     158        if content_length > settings.FILE_UPLOAD_MAX_MEMORY_SIZE:
     159            self.activated = False
     160        else:
     161            self.activated = True
     162
     163    def new_file(self, *args, **kwargs):
     164        super(MemoryFileUploadHandler, self).new_file(*args, **kwargs)
     165        if self.activated:
     166            self.file = StringIO()
     167            raise StopFutureHandlers()
     168
     169    def receive_data_chunk(self, raw_data, start):
     170        """
     171        Add the data to the StringIO file.
     172        """
     173        if self.activated:
     174            self.file.write(raw_data)
     175        else:
     176            return raw_data
     177
     178    def file_complete(self, file_size):
     179        """
     180        Return a file object if we're activated.
     181        """
     182        if not self.activated:
     183            return
     184
     185        return InMemoryUploadedFile(self.file, self.field_name, self.file_name,
     186                                    self.content_type, self.charset, file_size)
     187
     188class TemporaryFile(object):
     189    """
     190    A temporary file that tries to delete itself when garbage collected.
     191    """
     192    def __init__(self, dir):
     193        if not dir:
     194            dir = tempfile.gettempdir()
     195        try:
     196            (fd, name) = tempfile.mkstemp(suffix='.upload', dir=dir)
     197            self.file = os.fdopen(fd, 'w+b')
     198        except (OSError, IOError):
     199            raise OSError("Could not create temporary file for uploading, have you set settings.FILE_UPLOAD_TEMP_DIR correctly?")
     200        self.name = name
     201
     202    def __getattr__(self, name):
     203        a = getattr(self.__dict__['file'], name)
     204        if type(a) != type(0):
     205            setattr(self, name, a)
     206        return a
     207
     208    def __del__(self):
     209        try:
     210            os.unlink(self.name)
     211        except OSError:
     212            pass
     213
     214def load_handler(path, *args, **kwargs):
     215    """
     216    Given a path to a handler, return an instance of that handler.
     217
     218    E.g.::
     219        >>> load_handler('django.core.files.uploadhandler.TemporaryFileUploadHandler', request)
     220        <TemporaryFileUploadHandler object at 0x...>
     221
     222    """
     223    i = path.rfind('.')
     224    module, attr = path[:i], path[i+1:]
     225    try:
     226        mod = __import__(module, {}, {}, [attr])
     227    except ImportError, e:
     228        raise ImproperlyConfigured('Error importing upload handler module %s: "%s"' % (module, e))
     229    except ValueError, e:
     230        raise ImproperlyConfigured('Error importing upload handler module. Is FILE_UPLOAD_HANDLERS a correctly defined list or tuple?')
     231    try:
     232        cls = getattr(mod, attr)
     233    except AttributeError:
     234        raise ImproperlyConfigured('Module "%s" does not define a "%s" upload handler backend' % (module, attr))
     235    return cls(*args, **kwargs)
  • django/core/handlers/modpython.py

    diff --git a/django/core/handlers/modpython.py b/django/core/handlers/modpython.py
    index abab399..332df6f 100644
    a b class ModPythonRequest(http.HttpRequest): 
    5353    def _load_post_and_files(self):
    5454        "Populates self._post and self._files"
    5555        if 'content-type' in self._req.headers_in and self._req.headers_in['content-type'].startswith('multipart'):
    56             self._post, self._files = http.parse_file_upload(self._req.headers_in, self.raw_post_data)
     56            self._raw_post_data = ''
     57            self._post, self._files = self.parse_file_upload(self.META, self._req)
    5758        else:
    5859            self._post, self._files = http.QueryDict(self.raw_post_data, encoding=self._encoding), datastructures.MultiValueDict()
    5960
  • django/core/handlers/wsgi.py

    diff --git a/django/core/handlers/wsgi.py b/django/core/handlers/wsgi.py
    index df2ba19..795f139 100644
    a b class WSGIRequest(http.HttpRequest): 
    112112        # Populates self._post and self._files
    113113        if self.method == 'POST':
    114114            if self.environ.get('CONTENT_TYPE', '').startswith('multipart'):
    115                 header_dict = dict([(k, v) for k, v in self.environ.items() if k.startswith('HTTP_')])
    116                 header_dict['Content-Type'] = self.environ.get('CONTENT_TYPE', '')
    117                 self._post, self._files = http.parse_file_upload(header_dict, self.raw_post_data)
     115                self._raw_post_data = ''
     116                self._post, self._files = self.parse_file_upload(self.META, self.environ['wsgi.input'])
    118117            else:
    119118                self._post, self._files = http.QueryDict(self.raw_post_data, encoding=self._encoding), datastructures.MultiValueDict()
    120119        else:
  • django/db/models/base.py

    diff --git a/django/db/models/base.py b/django/db/models/base.py
    index 14a4804..24add11 100644
    a b from django.dispatch import dispatcher 
    1919from django.utils.datastructures import SortedDict
    2020from django.utils.functional import curry
    2121from django.utils.encoding import smart_str, force_unicode, smart_unicode
     22from django.core.files.move import file_move_safe
     23from django.core.files import locks
    2224from django.conf import settings
    2325
    2426try:
    class Model(object): 
    451453    def _get_FIELD_size(self, field):
    452454        return os.path.getsize(self._get_FIELD_filename(field))
    453455
    454     def _save_FIELD_file(self, field, filename, raw_contents, save=True):
     456    def _save_FIELD_file(self, field, filename, raw_field, save=True):
    455457        directory = field.get_directory_name()
    456458        try: # Create the date-based directory if it doesn't exist.
    457459            os.makedirs(os.path.join(settings.MEDIA_ROOT, directory))
    458460        except OSError: # Directory probably already exists.
    459461            pass
     462
     463        #
     464        # Check for old-style usage (files-as-dictionaries). Warn here first
     465        # since there are multiple locations where we need to support both new
     466        # and old usage.
     467        #
     468        if isinstance(raw_field, dict):
     469            import warnings
     470            warnings.warn(
     471                message = "Representing uploaded files as dictionaries is"\
     472                          " deprected. Use django.core.files.SimpleUploadedFile"\
     473                          " instead.",
     474                category = DeprecationWarning,
     475                stacklevel = 2
     476            )
     477            from django.core.files.uploadedfile import SimpleUploadedFile
     478            raw_field = SimpleUploadedFile.from_dict(raw_field)
     479
     480        elif isinstance(raw_field, basestring):
     481            import warnings
     482            warnings.warn(
     483                message = "Representing uploaded files as strings is "\
     484                          " deprecated. Use django.core.files.SimpleUploadedFile "\
     485                          " instead.",
     486                category = DeprecationWarning,
     487                stacklevel = 2
     488            )
     489            from django.core.files.uploadedfile import SimpleUploadedFile
     490            raw_field = SimpleUploadedFile(filename, raw_field)
     491
     492        if filename is None:
     493            filename = raw_field.file_name
     494
    460495        filename = field.get_filename(filename)
    461496
     497        #
    462498        # If the filename already exists, keep adding an underscore to the name of
    463499        # the file until the filename doesn't exist.
     500        #
    464501        while os.path.exists(os.path.join(settings.MEDIA_ROOT, filename)):
    465502            try:
    466503                dot_index = filename.rindex('.')
    class Model(object): 
    468505                filename += '_'
    469506            else:
    470507                filename = filename[:dot_index] + '_' + filename[dot_index:]
     508        #
     509        # Save the file name on the object and write the file to disk
     510        #
    471511
    472         # Write the file to disk.
    473512        setattr(self, field.attname, filename)
    474513
    475514        full_filename = self._get_FIELD_filename(field)
    476         fp = open(full_filename, 'wb')
    477         fp.write(raw_contents)
    478         fp.close()
     515
     516        if hasattr(raw_field, 'temporary_file_path'):
     517            # This file has a file path that we can move.
     518            raw_field.close()
     519            file_move_safe(raw_field.temporary_file_path(), full_filename)
     520
     521        else:
     522            # This is a normal uploadedfile that we can stream.
     523            fp = open(full_filename, 'wb')
     524            locks.lock(fp, locks.LOCK_EX)
     525            for chunk in raw_field.chunk():
     526                fp.write(chunk)
     527            locks.unlock(fp)
     528            fp.close()
    479529
    480530        # Save the width and/or height, if applicable.
    481531        if isinstance(field, ImageField) and (field.width_field or field.height_field):
  • django/db/models/fields/__init__.py

    diff --git a/django/db/models/fields/__init__.py b/django/db/models/fields/__init__.py
    index a893c25..d088729 100644
    a b class FileField(Field): 
    808808        setattr(cls, 'get_%s_filename' % self.name, curry(cls._get_FIELD_filename, field=self))
    809809        setattr(cls, 'get_%s_url' % self.name, curry(cls._get_FIELD_url, field=self))
    810810        setattr(cls, 'get_%s_size' % self.name, curry(cls._get_FIELD_size, field=self))
    811         setattr(cls, 'save_%s_file' % self.name, lambda instance, filename, raw_contents, save=True: instance._save_FIELD_file(self, filename, raw_contents, save))
     811        setattr(cls, 'save_%s_file' % self.name, lambda instance, filename, raw_field, save=True: instance._save_FIELD_file(self, filename, raw_field, save))
    812812        dispatcher.connect(self.delete_file, signal=signals.post_delete, sender=cls)
    813813
    814814    def delete_file(self, instance):
    class FileField(Field): 
    831831        if new_data.get(upload_field_name, False):
    832832            func = getattr(new_object, 'save_%s_file' % self.name)
    833833            if rel:
    834                 func(new_data[upload_field_name][0]["filename"], new_data[upload_field_name][0]["content"], save)
     834                file = new_data[upload_field_name][0]
    835835            else:
    836                 func(new_data[upload_field_name]["filename"], new_data[upload_field_name]["content"], save)
     836                file = new_data[upload_field_name]
     837
     838            # Backwards-compatible support for files-as-dictionaries.
     839            # We don't need to raise a warning because Model._save_FIELD_file will
     840            # do so for us.
     841            try:
     842                file_name = file.file_name
     843            except AttributeError:
     844                file_name = file['filename']
     845
     846            func(file_name, file, save)
    837847
    838848    def get_directory_name(self):
    839849        return os.path.normpath(force_unicode(datetime.datetime.now().strftime(smart_str(self.upload_to))))
    class FileField(Field): 
    846856    def save_form_data(self, instance, data):
    847857        from django.newforms.fields import UploadedFile
    848858        if data and isinstance(data, UploadedFile):
    849             getattr(instance, "save_%s_file" % self.name)(data.filename, data.content, save=False)
     859            getattr(instance, "save_%s_file" % self.name)(data.filename, data.data, save=False)
    850860
    851861    def formfield(self, **kwargs):
    852862        defaults = {'form_class': forms.FileField}
  • django/http/__init__.py

    diff --git a/django/http/__init__.py b/django/http/__init__.py
    index 7faa3c8..ef15479 100644
    a b try: 
    99except ImportError:
    1010    from cgi import parse_qsl
    1111
    12 from django.utils.datastructures import MultiValueDict, FileDict
     12from django.utils.datastructures import MultiValueDict, ImmutableList
    1313from django.utils.encoding import smart_str, iri_to_uri, force_unicode
    14 
     14from django.http.multipartparser import MultiPartParser
     15from django.conf import settings
     16from django.core.files import uploadhandler
    1517from utils import *
    1618
    1719RESERVED_CHARS="!*'();:@&=+$,/?%#[]"
    1820
    19 
    2021class Http404(Exception):
    2122    pass
    2223
    class HttpRequest(object): 
    2526
    2627    # The encoding used in GET/POST dicts. None means use default setting.
    2728    _encoding = None
     29    _upload_handlers = []
    2830
    2931    def __init__(self):
    3032        self.GET, self.POST, self.COOKIES, self.META, self.FILES = {}, {}, {}, {}, {}
    class HttpRequest(object): 
    102104
    103105    encoding = property(_get_encoding, _set_encoding)
    104106
    105 def parse_file_upload(header_dict, post_data):
    106     """Returns a tuple of (POST QueryDict, FILES MultiValueDict)."""
    107     import email, email.Message
    108     from cgi import parse_header
    109     raw_message = '\r\n'.join(['%s:%s' % pair for pair in header_dict.items()])
    110     raw_message += '\r\n\r\n' + post_data
    111     msg = email.message_from_string(raw_message)
    112     POST = QueryDict('', mutable=True)
    113     FILES = MultiValueDict()
    114     for submessage in msg.get_payload():
    115         if submessage and isinstance(submessage, email.Message.Message):
    116             name_dict = parse_header(submessage['Content-Disposition'])[1]
    117             # name_dict is something like {'name': 'file', 'filename': 'test.txt'} for file uploads
    118             # or {'name': 'blah'} for POST fields
    119             # We assume all uploaded files have a 'filename' set.
    120             if 'filename' in name_dict:
    121                 assert type([]) != type(submessage.get_payload()), "Nested MIME messages are not supported"
    122                 if not name_dict['filename'].strip():
    123                     continue
    124                 # IE submits the full path, so trim everything but the basename.
    125                 # (We can't use os.path.basename because that uses the server's
    126                 # directory separator, which may not be the same as the
    127                 # client's one.)
    128                 filename = name_dict['filename'][name_dict['filename'].rfind("\\")+1:]
    129                 FILES.appendlist(name_dict['name'], FileDict({
    130                     'filename': filename,
    131                     'content-type': 'Content-Type' in submessage and submessage['Content-Type'] or None,
    132                     'content': submessage.get_payload(),
    133                 }))
    134             else:
    135                 POST.appendlist(name_dict['name'], submessage.get_payload())
    136     return POST, FILES
    137 
     107    def _initialize_handlers(self):
     108        self._upload_handlers = [uploadhandler.load_handler(handler, self)
     109                                 for handler in settings.FILE_UPLOAD_HANDLERS]
     110
     111    def _set_upload_handlers(self, upload_handlers):
     112        if hasattr(self, '_files'):
     113            raise AttributeError("You cannot set the upload handlers after the upload has been processed.")
     114        self._upload_handlers = upload_handlers
     115
     116    def _get_upload_handlers(self):
     117        if not self._upload_handlers:
     118            # If thre are no upload handlers defined, initialize them from settings.
     119            self._initialize_handlers()
     120        return self._upload_handlers
     121
     122    upload_handlers = property(_get_upload_handlers, _set_upload_handlers)
     123
     124    def parse_file_upload(self, META, post_data):
     125        """Returns a tuple of (POST QueryDict, FILES MultiValueDict)."""
     126        self.upload_handlers = ImmutableList(
     127            self.upload_handlers,
     128            warning = "You cannot alter upload handlers after the upload has been processed."
     129        )
     130        parser = MultiPartParser(META, post_data, self.upload_handlers, self.encoding)
     131        return parser.parse()
    138132
    139133class QueryDict(MultiValueDict):
    140134    """
  • new file django/http/multipartparser.py

    diff --git a/django/http/multipartparser.py b/django/http/multipartparser.py
    new file mode 100644
    index 0000000..8bed568
    - +  
     1"""
     2Multi-part parsing for file uploads.
     3
     4Exposes one class, ``MultiPartParser``, which feeds chunks of uploaded data to
     5file upload handlers for processing.
     6"""
     7import cgi
     8from django.conf import settings
     9from django.core.exceptions import SuspiciousOperation
     10from django.utils.datastructures import MultiValueDict
     11from django.utils.encoding import force_unicode
     12from django.utils.text import unescape_entities
     13from django.core.files.uploadhandler import StopUpload, SkipFile, StopFutureHandlers
     14
     15__all__ = ('MultiPartParser','MultiPartParserError','InputStreamExhausted')
     16
     17class MultiPartParserError(Exception):
     18    pass
     19
     20class InputStreamExhausted(Exception):
     21    """
     22    No more reads are allowed from this device.
     23    """
     24    pass
     25
     26RAW = "raw"
     27FILE = "file"
     28FIELD = "field"
     29
     30class MultiPartParser(object):
     31    """
     32    A rfc2388 multipart/form-data parser.
     33
     34    ``MultiValueDict.parse()`` reads the input stream in ``chunk_size`` chunks
     35    and returns a tuple of ``(MultiValueDict(POST), MultiValueDict(FILES))``. If
     36    ``file_upload_dir`` is defined files will be streamed to temporary files in
     37    that directory.
     38    """
     39    def __init__(self, META, input_data, upload_handlers, encoding=None):
     40        """
     41        Initialize the MultiPartParser object.
     42
     43        :META:
     44            The standard ``META`` dictionary in Django request objects.
     45        :input_data:
     46            The raw post data, as a bytestring.
     47        :upload_handler:
     48            An UploadHandler instance that performs operations on the uploaded
     49            data.
     50        :encoding:
     51            The encoding with which to treat the incoming data.
     52        """
     53
     54        #
     55        # Content-Type should containt multipart and the boundary information.
     56        #
     57
     58        content_type = META.get('HTTP_CONTENT_TYPE', META.get('CONTENT_TYPE', ''))
     59        if not content_type.startswith('multipart/'):
     60            raise MultiPartParserError('Invalid Content-Type: %s' % content_type)
     61
     62        # Parse the header to get the boundary to split the parts.
     63        ctypes, opts = parse_header(content_type)
     64        boundary = opts.get('boundary')
     65        if not boundary or not cgi.valid_boundary(boundary):
     66            raise MultiPartParserError('Invalid boundary in multipart: %s' % boundary)
     67
     68
     69        #
     70        # Content-Length should contain the length of the body we are about
     71        # to receive.
     72        #
     73        try:
     74            content_length = int(META.get('HTTP_CONTENT_LENGTH', META.get('CONTENT_LENGTH',0)))
     75        except (ValueError, TypeError):
     76            # For now set it to 0; we'll try again later on down.
     77            content_length = 0
     78
     79        if content_length <= 0:
     80            # This means we shouldn't continue...raise an error.
     81            raise MultiPartParserError("Invalid content length: %r" % content_length)
     82
     83        self._boundary = boundary
     84        self._input_data = input_data
     85
     86        # For compatibility with low-level network APIs (with 32-bit integers),
     87        # the chunk size should be < 2^31, but still divisible by 4.
     88        self._chunk_size = min(2**31-4, *[x.chunk_size for x in upload_handlers if x.chunk_size])
     89
     90        self._meta = META
     91        self._encoding = encoding or settings.DEFAULT_CHARSET
     92        self._content_length = content_length
     93        self._upload_handlers = upload_handlers
     94
     95    def parse(self):
     96        """
     97        Parse the POST data and break it into a FILES MultiValueDict and a POST
     98        MultiValueDict.
     99
     100        Returns a tuple containing the POST and FILES dictionary, respectively.
     101        """
     102        # We have to import QueryDict down here to avoid a circular import.
     103        from django.http import QueryDict
     104
     105        encoding = self._encoding
     106        handlers = self._upload_handlers
     107
     108        limited_input_data = LimitBytes(self._input_data, self._content_length)
     109
     110        # See if the handler will want to take care of the parsing.
     111        # This allows overriding everything if somebody wants it.
     112        for handler in handlers:
     113            result = handler.handle_raw_input(limited_input_data,
     114                                              self._meta,
     115                                              self._content_length,
     116                                              self._boundary,
     117                                              encoding)
     118            if result is not None:
     119                return result[0], result[1]
     120
     121        # Create the data structures to be used later.
     122        self._post = QueryDict('', mutable=True)
     123        self._files = MultiValueDict()
     124
     125        # Instantiate the parser and stream:
     126        stream = LazyStream(ChunkIter(limited_input_data, self._chunk_size))
     127
     128        # Whether or not to signal a file-completion at the beginning of the loop.
     129        old_field_name = None
     130        counters = [0] * len(handlers)
     131
     132        try:
     133            for item_type, meta_data, field_stream in Parser(stream, self._boundary):
     134                if old_field_name:
     135                    # We run this at the beginning of the next loop
     136                    # since we cannot be sure a file is complete until
     137                    # we hit the next boundary/part of the multipart content.
     138                    self.handle_file_complete(old_field_name, counters)
     139
     140                try:
     141                    disposition = meta_data['content-disposition'][1]
     142                    field_name = disposition['name'].strip()
     143                except (KeyError, IndexError, AttributeError):
     144                    continue
     145
     146                transfer_encoding = meta_data.get('content-transfer-encoding')
     147                field_name = force_unicode(field_name, encoding, errors='replace')
     148
     149                if item_type == FIELD:
     150                    # This is a post field, we can just set it in the post
     151                    if transfer_encoding == 'base64':
     152                        raw_data = field_stream.read()
     153                        try:
     154                            data = str(raw_data).decode('base64')
     155                        except:
     156                            data = raw_data
     157                    else:
     158                        data = field_stream.read()
     159
     160                    self._post.appendlist(field_name,
     161                                          force_unicode(data, encoding, errors='replace'))
     162                elif item_type == FILE:
     163                    # This is a file, use the handler...
     164                    file_successful = True
     165                    file_name = disposition.get('filename')
     166                    if not file_name:
     167                        continue
     168                    file_name = force_unicode(file_name, encoding, errors='replace')
     169                    file_name = self.IE_sanitize(unescape_entities(file_name))
     170
     171                    content_type = meta_data.get('content-type', ('',))[0].strip()
     172                    try:
     173                        charset = meta_data.get('content-type', (0,{}))[1].get('charset', None)
     174                    except:
     175                        charset = None
     176
     177                    try:
     178                        content_length = int(meta_data.get('content-length')[0])
     179                    except (IndexError, TypeError, ValueError):
     180                        content_length = None
     181
     182                    counters = [0] * len(handlers)
     183                    try:
     184                        for handler in handlers:
     185                            try:
     186                                handler.new_file(field_name, file_name,
     187                                                 content_type, content_length,
     188                                                 charset)
     189                            except StopFutureHandlers:
     190                                break
     191
     192                        for chunk in field_stream:
     193                            if transfer_encoding == 'base64':
     194                                # We only special-case base64 transfer encoding
     195                                try:
     196                                    chunk = str(chunk).decode('base64')
     197                                except Exception, e:
     198                                    # Since this is only a chunk, any error is an unfixable error.
     199                                    raise MultiPartParserError("Could not decode base64 data: %r" % e)
     200
     201                            for i, handler in enumerate(handlers):
     202                                chunk_length = len(chunk)
     203                                chunk = handler.receive_data_chunk(chunk,
     204                                                                   counters[i])
     205                                counters[i] += chunk_length
     206                                if chunk is None:
     207                                    # If the chunk received by the handler is None, then don't continue.
     208                                    break
     209
     210                    except SkipFile, e:
     211                        file_successful = False
     212                        # Just use up the rest of this file...
     213                        exhaust(field_stream)
     214                    else:
     215                        # Handle file upload completions on next iteration.
     216                        old_field_name = field_name
     217                else:
     218                    # If this is neither a FIELD or a FILE, just exhaust the stream.
     219                    exhaust(stream)
     220        except StopUpload, e:
     221            if not e.connection_reset:
     222                exhaust(limited_input_data)
     223        else:
     224            # Make sure that the request data is all fed
     225            exhaust(limited_input_data)
     226
     227        # Signal that the upload has completed.
     228        for handler in handlers:
     229            retval = handler.upload_complete()
     230            if retval:
     231                break
     232
     233        return self._post, self._files
     234
     235    def handle_file_complete(self, old_field_name, counters):
     236        """
     237        Handle all the signalling that takes place when a file is complete.
     238        """
     239        for i, handler in enumerate(self._upload_handlers):
     240            file_obj = handler.file_complete(counters[i])
     241            if file_obj:
     242                # If it returns a file object, then set the files dict.
     243                self._files.appendlist(force_unicode(old_field_name,
     244                                                     self._encoding,
     245                                                     errors='replace'),
     246                                       file_obj)
     247                break
     248
     249    def IE_sanitize(self, filename):
     250        """Cleanup filename from Internet Explorer full paths."""
     251        return filename and filename[filename.rfind("\\")+1:].strip()
     252
     253class LazyStream(object):
     254    """
     255    The LazyStream wrapper allows one to get and "unget" bytes from a stream.
     256
     257    Given a producer object (an iterator that yields bytestrings), the
     258    LazyStream object will support iteration, reading, and keeping a "look-back"
     259    variable in case you need to "unget" some bytes.
     260    """
     261    def __init__(self, producer, length=None):
     262        """
     263        Every LazyStream must have a producer when instantiated.
     264
     265        A producer is an iterable that returns a string each time it
     266        is called.
     267        """
     268        self._producer = producer
     269        self._empty = False
     270        self._leftover = ''
     271        self.length = length
     272        self._position = 0
     273        self._remaining = length
     274
     275        # These fields are to do sanity checking to make sure we don't
     276        # have infinite loops getting/ungetting from the stream. The
     277        # purpose overall is to raise an exception if we perform lots
     278        # of stream get/unget gymnastics without getting
     279        # anywhere. Naturally this is not sound, but most probably
     280        # would indicate a bug if the exception is raised.
     281
     282        # largest position tell us how far this lazystream has ever
     283        # been advanced
     284        self._largest_position = 0
     285
     286        # "modifications since" will start at zero and increment every
     287        # time the position is modified but a new largest position is
     288        # not achieved.
     289        self._modifications_since = 0
     290
     291    def tell(self):
     292        return self.position
     293
     294    def read(self, size=None):
     295        def parts():
     296            remaining = (size is not None and [size] or [self._remaining])[0]
     297            # do the whole thing in one shot if no limit was provided.
     298            if remaining is None:
     299                yield ''.join(self)
     300                return
     301
     302            # otherwise do some bookkeeping to return exactly enough
     303            # of the stream and stashing any extra content we get from
     304            # the producer
     305            while remaining != 0:
     306                assert remaining > 0, 'remaining bytes to read should never go negative'
     307
     308                chunk = self.next()
     309
     310                emitting = chunk[:remaining]
     311                self.unget(chunk[remaining:])
     312                remaining -= len(emitting)
     313                yield emitting
     314
     315        out = ''.join(parts())
     316        return out
     317
     318    def next(self):
     319        """
     320        Used when the exact number of bytes to read is unimportant.
     321
     322        This procedure just returns whatever is chunk is conveniently returned
     323        from the iterator instead. Useful to avoid unnecessary bookkeeping if
     324        performance is an issue.
     325        """
     326        if self._leftover:
     327            output = self._leftover
     328            self._leftover = ''
     329        else:
     330            output = self._producer.next()
     331        self.position += len(output)
     332        return output
     333
     334    def close(self):
     335        """
     336        Used to invalidate/disable this lazy stream.
     337
     338        Replaces the producer with an empty list. Any leftover bytes that have
     339        already been read will still be reported upon read() and/or next().
     340        """
     341        self._producer = []
     342
     343    def __iter__(self):
     344        return self
     345
     346    def unget(self, bytes):
     347        """
     348        Places bytes back onto the front of the lazy stream.
     349
     350        Future calls to read() will return those bytes first. The
     351        stream position and thus tell() will be rewound.
     352        """
     353        self.position -= len(bytes)
     354        self._leftover = ''.join([bytes, self._leftover])
     355
     356    def _set_position(self, value):
     357        if value > self._largest_position:
     358            self._modifications_since = 0
     359            self._largest_position = value
     360        else:
     361            self._modifications_since += 1
     362            if self._modifications_since > 500:
     363                raise SuspiciousOperation(
     364                    "The multipart parser got stuck, which shouldn't happen with"
     365                    " normal uploaded files. Check for malicious upload activity;"
     366                    " if there is none, report this to the Django developers."
     367                )
     368
     369        self._position = value
     370
     371    position = property(lambda self: self._position, _set_position)
     372
     373class ChunkIter(object):
     374    """
     375    An iterable that will yield chunks of data. Given a file-like object as the
     376    constructor, this object will yield chunks of read operations from that
     377    object.
     378    """
     379    def __init__(self, flo, chunk_size=64 * 1024):
     380        self.flo = flo
     381        self.chunk_size = chunk_size
     382
     383    def next(self):
     384        try:
     385            data = self.flo.read(self.chunk_size)
     386        except InputStreamExhausted:
     387            raise StopIteration()
     388        if data:
     389            return data
     390        else:
     391            raise StopIteration()
     392
     393    def __iter__(self):
     394        return self
     395
     396class LimitBytes(object):
     397    """ Limit bytes for a file object. """
     398    def __init__(self, fileobject, length):
     399        self._file = fileobject
     400        self.remaining = length
     401
     402    def read(self, num_bytes=None):
     403        """
     404        Read data from the underlying file.
     405        If you ask for too much or there isn't anything left,
     406        this will raise an InputStreamExhausted error.
     407        """
     408        if self.remaining <= 0:
     409            raise InputStreamExhausted()
     410        if num_bytes is None:
     411            num_bytes = self.remaining
     412        else:
     413            num_bytes = min(num_bytes, self.remaining)
     414        self.remaining -= num_bytes
     415        return self._file.read(num_bytes)
     416
     417class InterBoundaryIter(object):
     418    """
     419    A Producer that will iterate over boundaries.
     420    """
     421    def __init__(self, stream, boundary):
     422        self._stream = stream
     423        self._boundary = boundary
     424
     425    def __iter__(self):
     426        return self
     427
     428    def next(self):
     429        try:
     430            return LazyStream(BoundaryIter(self._stream, self._boundary))
     431        except InputStreamExhausted:
     432            raise StopIteration()
     433
     434class BoundaryIter(object):
     435    """
     436    A Producer that is sensitive to boundaries.
     437
     438    Will happily yield bytes until a boundary is found. Will yield the bytes
     439    before the boundary, throw away the boundary bytes themselves, and push the
     440    post-boundary bytes back on the stream.
     441
     442    The future calls to .next() after locating the boundary will raise a
     443    StopIteration exception.
     444    """
     445
     446    def __init__(self, stream, boundary):
     447        self._stream = stream
     448        self._boundary = boundary
     449        self._done = False
     450        # rollback an additional six bytes because the format is like
     451        # this: CRLF<boundary>[--CRLF]
     452        self._rollback = len(boundary) + 6
     453
     454        # Try to use mx fast string search if available. Otherwise
     455        # use Python find. Wrap the latter for consistency.
     456        unused_char = self._stream.read(1)
     457        if not unused_char:
     458            raise InputStreamExhausted()
     459        self._stream.unget(unused_char)
     460        try:
     461            from mx.TextTools import FS
     462            self._fs = FS(boundary).find
     463        except ImportError:
     464            self._fs = lambda data: data.find(boundary)
     465
     466    def __iter__(self):
     467        return self
     468
     469    def next(self):
     470        if self._done:
     471            raise StopIteration()
     472
     473        stream = self._stream
     474        rollback = self._rollback
     475
     476        bytes_read = 0
     477        chunks = []
     478        for bytes in stream:
     479            bytes_read += len(bytes)
     480            chunks.append(bytes)
     481            if bytes_read > rollback:
     482                break
     483            if not bytes:
     484                break
     485        else:
     486            self._done = True
     487
     488        if not chunks:
     489            raise StopIteration()
     490
     491        chunk = ''.join(chunks)
     492        boundary = self._find_boundary(chunk, len(chunk) < self._rollback)
     493
     494        if boundary:
     495            end, next = boundary
     496            stream.unget(chunk[next:])
     497            self._done = True
     498            return chunk[:end]
     499        else:
     500            # make sure we dont treat a partial boundary (and
     501            # its separators) as data
     502            if not chunk[:-rollback]:# and len(chunk) >= (len(self._boundary) + 6):
     503                # There's nothing left, we should just return and mark as done.
     504                self._done = True
     505                return chunk
     506            else:
     507                stream.unget(chunk[-rollback:])
     508                return chunk[:-rollback]
     509
     510    def _find_boundary(self, data, eof = False):
     511        """
     512        Finds a multipart boundary in data.
     513
     514        Should no boundry exist in the data None is returned instead. Otherwise
     515        a tuple containing the indices of the following are returned:
     516
     517         * the end of current encapsulation
     518         * the start of the next encapsulation
     519        """
     520        index = self._fs(data)
     521        if index < 0:
     522            return None
     523        else:
     524            end = index
     525            next = index + len(self._boundary)
     526            data_len = len(data) - 1
     527            # backup over CRLF
     528            if data[max(0,end-1)] == '\n':
     529                end -= 1
     530            if data[max(0,end-1)] == '\r':
     531                end -= 1
     532            # skip over --CRLF
     533            #if data[min(data_len,next)] == '-':
     534            #    next += 1
     535            #if data[min(data_len,next)] == '-':
     536            #    next += 1
     537            #if data[min(data_len,next)] == '\r':
     538            #    next += 1
     539            #if data[min(data_len,next)] == '\n':
     540            #    next += 1
     541            return end, next
     542
     543def exhaust(stream_or_iterable):
     544    """
     545    Completely exhausts an iterator or stream.
     546
     547    Raise a MultiPartParserError if the argument is not a stream or an iterable.
     548    """
     549    iterator = None
     550    try:
     551        iterator = iter(stream_or_iterable)
     552    except TypeError:
     553        iterator = ChunkIter(stream_or_iterable, 16384)
     554
     555    if iterator is None:
     556        raise MultiPartParserError('multipartparser.exhaust() was passed a non-iterable or stream parameter')
     557
     558    for __ in iterator:
     559        pass
     560
     561def parse_boundary_stream(stream, max_header_size):
     562    """
     563    Parses one and exactly one stream that encapsulates a boundary.
     564    """
     565    # Stream at beginning of header, look for end of header
     566    # and parse it if found. The header must fit within one
     567    # chunk.
     568    chunk = stream.read(max_header_size)
     569
     570    # 'find' returns the top of these four bytes, so we'll
     571    # need to munch them later to prevent them from polluting
     572    # the payload.
     573    header_end = chunk.find('\r\n\r\n')
     574
     575    def _parse_header(line):
     576        main_value_pair, params = parse_header(line)
     577        try:
     578            name, value = main_value_pair.split(':', 1)
     579        except:
     580            raise ValueError("Invalid header: %r" % line)
     581        return name, (value, params)
     582
     583    if header_end == -1:
     584        # we find no header, so we just mark this fact and pass on
     585        # the stream verbatim
     586        stream.unget(chunk)
     587        return (RAW, {}, stream)
     588
     589    header = chunk[:header_end]
     590
     591    # here we place any excess chunk back onto the stream, as
     592    # well as throwing away the CRLFCRLF bytes from above.
     593    stream.unget(chunk[header_end + 4:])
     594
     595    TYPE = RAW
     596    outdict = {}
     597
     598    # Eliminate blank lines
     599    for line in header.split('\r\n'):
     600        # This terminology ("main value" and "dictionary of
     601        # parameters") is from the Python docs.
     602        try:
     603            name, (value, params) = _parse_header(line)
     604        except:
     605            continue
     606
     607        if name == 'content-disposition':
     608            TYPE = FIELD
     609            if params.get('filename'):
     610                TYPE = FILE
     611
     612        outdict[name] = value, params
     613
     614    if TYPE == RAW:
     615        stream.unget(chunk)
     616
     617    return (TYPE, outdict, stream)
     618
     619class Parser(object):
     620    def __init__(self, stream, boundary):
     621        self._stream = stream
     622        self._separator = '--' + boundary
     623
     624    def __iter__(self):
     625        boundarystream = InterBoundaryIter(self._stream, self._separator)
     626        for sub_stream in boundarystream:
     627            # Iterate over each part
     628            yield parse_boundary_stream(sub_stream, 1024)
     629
     630def parse_header(line):
     631    """ Parse the header into a key-value. """
     632    plist = _parse_header_params(';' + line)
     633    key = plist.pop(0).lower()
     634    pdict = {}
     635    for p in plist:
     636        i = p.find('=')
     637        if i >= 0:
     638            name = p[:i].strip().lower()
     639            value = p[i+1:].strip()
     640            if len(value) >= 2 and value[0] == value[-1] == '"':
     641                value = value[1:-1]
     642                value = value.replace('\\\\', '\\').replace('\\"', '"')
     643            pdict[name] = value
     644    return key, pdict
     645
     646def _parse_header_params(s):
     647    plist = []
     648    while s[:1] == ';':
     649        s = s[1:]
     650        end = s.find(';')
     651        while end > 0 and s.count('"', 0, end) % 2:
     652            end = s.find(';', end + 1)
     653        if end < 0:
     654            end = len(s)
     655        f = s[:end]
     656        plist.append(f.strip())
     657        s = s[end:]
     658    return plist
  • django/newforms/fields.py

    diff --git a/django/newforms/fields.py b/django/newforms/fields.py
    index dfe46a2..7d73396 100644
    a b import datetime 
    77import os
    88import re
    99import time
     10try:
     11    from cStringIO import StringIO
     12except ImportError:
     13    from StringIO import StringIO
     14
    1015# Python 2.3 fallbacks
    1116try:
    1217    from decimal import Decimal, DecimalException
    except ImportError: 
    416421
    417422class UploadedFile(StrAndUnicode):
    418423    "A wrapper for files uploaded in a FileField"
    419     def __init__(self, filename, content):
     424    def __init__(self, filename, data):
    420425        self.filename = filename
    421         self.content = content
     426        self.data = data
    422427
    423428    def __unicode__(self):
    424429        """
    class FileField(Field): 
    444449            return None
    445450        elif not data and initial:
    446451            return initial
     452
     453        if isinstance(data, dict):
     454            # We warn once, then support both ways below.
     455            import warnings
     456            warnings.warn(
     457                message = "Representing uploaded files as dictionaries is"\
     458                          " deprecated. Use django.core.files.SimpleUploadedFile "\
     459                          " instead.",
     460                category = DeprecationWarning,
     461                stacklevel = 2
     462            )
     463
    447464        try:
    448             f = UploadedFile(data['filename'], data['content'])
    449         except TypeError:
     465            file_name = data.file_name
     466            file_size = data.file_size
     467        except AttributeError:
     468            try:
     469                file_name = data.get('filename')
     470                file_size = bool(data['content'])
     471            except (AttributeError, KeyError):
     472                raise ValidationError(self.error_messages['invalid'])
     473
     474        if not file_name:
    450475            raise ValidationError(self.error_messages['invalid'])
    451         except KeyError:
    452             raise ValidationError(self.error_messages['missing'])
    453         if not f.content:
     476        if not file_size:
    454477            raise ValidationError(self.error_messages['empty'])
    455         return f
     478
     479        return UploadedFile(file_name, data)
    456480
    457481class ImageField(FileField):
    458482    default_error_messages = {
    class ImageField(FileField): 
    470494        elif not data and initial:
    471495            return initial
    472496        from PIL import Image
    473         from cStringIO import StringIO
     497
     498        # We need to get a file object for PIL. We might have a path or we might
     499        # have to read the data into memory.
     500        if hasattr(data, 'temporary_file_path'):
     501            file = data.temporary_file_path()
     502        else:
     503            if hasattr(data, 'read'):
     504                file = StringIO(data.read())
     505            else:
     506                file = StringIO(data['content'])
     507
    474508        try:
    475509            # load() is the only method that can spot a truncated JPEG,
    476510            #  but it cannot be called sanely after verify()
    477             trial_image = Image.open(StringIO(f.content))
     511            trial_image = Image.open(file)
    478512            trial_image.load()
     513
     514            # Since we're about to use the file again we have to reset the
     515            # file object if possible.
     516            if hasattr(file, 'reset'):
     517                file.reset()
     518
    479519            # verify() is the only method that can spot a corrupt PNG,
    480520            #  but it must be called immediately after the constructor
    481             trial_image = Image.open(StringIO(f.content))
     521            trial_image = Image.open(file)
    482522            trial_image.verify()
    483523        except Exception: # Python Imaging Library doesn't recognize it as an image
    484524            raise ValidationError(self.error_messages['invalid_image'])
  • django/oldforms/__init__.py

    diff --git a/django/oldforms/__init__.py b/django/oldforms/__init__.py
    index fc87271..ee838d2 100644
    a b class FileUploadField(FormField): 
    680680        self.field_name, self.is_required = field_name, is_required
    681681        self.validator_list = [self.isNonEmptyFile] + validator_list
    682682
    683     def isNonEmptyFile(self, field_data, all_data):
     683    def isNonEmptyFile(self, new_data, all_data):
     684        if hasattr(new_data, 'upload_errors'):
     685            upload_errors = new_data.upload_errors()
     686            if upload_errors:
     687                raise validators.CriticalValidationError, upload_errors
    684688        try:
    685             content = field_data['content']
    686         except TypeError:
    687             raise validators.CriticalValidationError, ugettext("No file was submitted. Check the encoding type on the form.")
    688         if not content:
     689            file_size = new_data.file_size
     690        except AttributeError:
     691            file_size = len(new_data['content'])
     692        if not file_size:
    689693            raise validators.CriticalValidationError, ugettext("The submitted file is empty.")
    690694
    691695    def render(self, data):
    692696        return mark_safe(u'<input type="file" id="%s" class="v%s" name="%s" />' % \
    693697            (self.get_id(), self.__class__.__name__, self.field_name))
    694698
     699    def prepare(self, new_data):
     700        if hasattr(new_data, 'upload_errors'):
     701            upload_errors = new_data.upload_errors()
     702            new_data[self.field_name] = { '_file_upload_error': upload_errors }
     703
    695704    def html2python(data):
    696705        if data is None:
    697706            raise EmptyValue
  • django/test/client.py

    diff --git a/django/test/client.py b/django/test/client.py
    index a15876e..6313181 100644
    a b  
    11import urllib
    22import sys
    33import os
    4 from cStringIO import StringIO
     4try:
     5    from cStringIO import StringIO
     6except ImportError:
     7    from StringIO import StringIO
    58from django.conf import settings
    69from django.contrib.auth import authenticate, login
    710from django.core.handlers.base import BaseHandler
    from django.utils.itercompat import is_iterable 
    1922BOUNDARY = 'BoUnDaRyStRiNg'
    2023MULTIPART_CONTENT = 'multipart/form-data; boundary=%s' % BOUNDARY
    2124
     25class FakePayload(object):
     26    """
     27    A wrapper around StringIO that restricts what can be read since data from
     28    the network can't be seeked and cannot be read outside of its content
     29    length. This makes sure that views can't do anything under the test client
     30    that wouldn't work in Real Life.
     31    """
     32    def __init__(self, content):
     33        self.__content = StringIO(content)
     34        self.__len = len(content)
     35
     36    def read(self, num_bytes=None):
     37        if num_bytes is None:
     38            num_bytes = self.__len or 1
     39        assert self.__len >= num_bytes, "Cannot read more than the available bytes from the HTTP incoming data."
     40        content = self.__content.read(num_bytes)
     41        self.__len -= num_bytes
     42        return content
     43
    2244class ClientHandler(BaseHandler):
    2345    """
    2446    A HTTP Handler that can be used for testing purposes.
    class Client: 
    236258            'CONTENT_TYPE':   content_type,
    237259            'PATH_INFO':      urllib.unquote(path),
    238260            'REQUEST_METHOD': 'POST',
    239             'wsgi.input':     StringIO(post_data),
     261            'wsgi.input':     FakePayload(post_data),
    240262        }
    241263        r.update(extra)
    242264
  • django/utils/datastructures.py

    diff --git a/django/utils/datastructures.py b/django/utils/datastructures.py
    index 21a72f2..f27bc1c 100644
    a b class DotExpandedDict(dict): 
    332332            except TypeError: # Special-case if current isn't a dict.
    333333                current = {bits[-1]: v}
    334334
    335 class FileDict(dict):
     335class ImmutableList(tuple):
    336336    """
    337     A dictionary used to hold uploaded file contents. The only special feature
    338     here is that repr() of this object won't dump the entire contents of the
    339     file to the output. A handy safeguard for a large file upload.
     337    A tuple-like object that raises useful errors when it is asked to mutate.
     338
     339    Example::
     340
     341        >>> a = ImmutableList(range(5), warning="You cannot mutate this.")
     342        >>> a[3] = '4'
     343        Traceback (most recent call last):
     344            ...
     345        AttributeError: You cannot mutate this.
    340346    """
    341     def __repr__(self):
    342         if 'content' in self:
    343             d = dict(self, content='<omitted>')
    344             return dict.__repr__(d)
    345         return dict.__repr__(self)
     347
     348    def __new__(cls, *args, **kwargs):
     349        if 'warning' in kwargs:
     350            warning = kwargs['warning']
     351            del kwargs['warning']
     352        else:
     353            warning = 'ImmutableList object is immutable.'
     354        self = tuple.__new__(cls, *args, **kwargs)
     355        self.warning = warning
     356        return self
     357
     358    def complain(self, *wargs, **kwargs):
     359        if isinstance(self.warning, Exception):
     360            raise self.warning
     361        else:
     362            raise AttributeError, self.warning
     363
     364    # All list mutation functions complain.
     365    __delitem__  = complain
     366    __delslice__ = complain
     367    __iadd__     = complain
     368    __imul__     = complain
     369    __setitem__  = complain
     370    __setslice__ = complain
     371    append       = complain
     372    extend       = complain
     373    insert       = complain
     374    pop          = complain
     375    remove       = complain
     376    sort         = complain
     377    reverse      = complain
    346378
    347379class DictWrapper(dict):
    348380    """
  • django/utils/text.py

    diff --git a/django/utils/text.py b/django/utils/text.py
    index aa190c8..3686a45 100644
    a b from django.conf import settings 
    33from django.utils.encoding import force_unicode
    44from django.utils.functional import allow_lazy
    55from django.utils.translation import ugettext_lazy
     6from htmlentitydefs import name2codepoint
    67
    78# Capitalizes the first letter of a string.
    89capfirst = lambda x: x and force_unicode(x)[0].upper() + force_unicode(x)[1:]
    def smart_split(text): 
    222223            yield bit
    223224smart_split = allow_lazy(smart_split, unicode)
    224225
     226def _replace_entity(match):
     227     text = match.group(1)
     228     if text[0] == u'#':
     229         text = text[1:]
     230         try:
     231             if text[0] in u'xX':
     232                 c = int(text[1:], 16)
     233             else:
     234                 c = int(text)
     235             return unichr(c)
     236         except ValueError:
     237             return match.group(0)
     238     else:
     239         try:
     240             return unichr(name2codepoint[text])
     241         except (ValueError, KeyError):
     242             return match.group(0)
     243
     244_entity_re = re.compile(r"&(#?[xX]?(?:[0-9a-fA-F]+|\w{1,8}));")
     245
     246def unescape_entities(text):
     247     return _entity_re.sub(_replace_entity, text)
     248unescape_entities = allow_lazy(unescape_entities, unicode)
  • docs/newforms.txt

    diff --git a/docs/newforms.txt b/docs/newforms.txt
    index 04e4c1a..296fc04 100644
    a b ContactForm to include an ``ImageField`` called ``mugshot``, we 
    805805need to bind the file data containing the mugshot image::
    806806
    807807    # Bound form with an image field
     808    >>> from django.core.files.uploadedfile import SimpleUploadedFile
    808809    >>> data = {'subject': 'hello',
    809810    ...         'message': 'Hi there',
    810811    ...         'sender': 'foo@example.com',
    811812    ...         'cc_myself': True}
    812     >>> file_data = {'mugshot': {'filename':'face.jpg'
    813     ...                          'content': <file data>}}
     813    >>> file_data = {'mugshot': SimpleUploadedFile('face.jpg', <file data>)}
    814814    >>> f = ContactFormWithMugshot(data, file_data)
    815815
    816816In practice, you will usually specify ``request.FILES`` as the source
  • docs/request_response.txt

    diff --git a/docs/request_response.txt b/docs/request_response.txt
    index 866a697..54fc24d 100644
    a b All attributes except ``session`` should be considered read-only. 
    8080    strings.
    8181
    8282``FILES``
     83   
     84    .. admonition:: Changed in Django development version
     85       
     86        In previous versions of Django, ``request.FILES`` contained
     87        simple ``dict`` objects representing uploaded files. This is
     88        no longer true -- files are represented by ``UploadedFile``
     89        objects as described below.
     90       
     91        These ``UploadedFile`` objects will emulate the old-style ``dict``
     92        interface, but this is deprecated and will be removed in the next
     93        release of Django.
     94       
    8395    A dictionary-like object containing all uploaded files. Each key in
    8496    ``FILES`` is the ``name`` from the ``<input type="file" name="" />``. Each
    85     value in ``FILES`` is a standard Python dictionary with the following three
    86     keys:
     97    value in ``FILES`` is an ``UploadedFile`` object containing the following
     98    attributes:
    8799
    88         * ``filename`` -- The name of the uploaded file, as a Python string.
    89         * ``content-type`` -- The content type of the uploaded file.
    90         * ``content`` -- The raw content of the uploaded file.
     100        * ``read(num_bytes=None)`` -- Read a number of bytes from the file.
     101        * ``file_name`` -- The name of the uploaded file.
     102        * ``file_size`` -- The size, in bytes, of the uploaded file.
     103        * ``chunk()`` -- A generator that yields sequential chunks of data.
    91104
     105    See `File Uploads`_ for more information.
     106   
    92107    Note that ``FILES`` will only contain data if the request method was POST
    93108    and the ``<form>`` that posted to the request had
    94109    ``enctype="multipart/form-data"``. Otherwise, ``FILES`` will be a blank
    95110    dictionary-like object.
     111   
     112    .. _File Uploads: ../upload_handling/
    96113
    97114``META``
    98115    A standard Python dictionary containing all available HTTP headers.
  • docs/settings.txt

    diff --git a/docs/settings.txt b/docs/settings.txt
    index 3fe999d..a68d2ff 100644
    a b Default: ``''`` (Empty string) 
    279279
    280280The database backend to use. The build-in database backends are
    281281``'postgresql_psycopg2'``, ``'postgresql'``, ``'mysql'``, ``'mysql_old'``,
    282 ``'sqlite3'`` and ``'oracle'``.
     282``'sqlite3'``, ``'oracle'``, and ``'oracle'``.
    283283
    284284In the Django development version, you can use a database backend that doesn't
    285285ship with Django by setting ``DATABASE_ENGINE`` to a fully-qualified path (i.e.
    Default: ``'utf-8'`` 
    530530The character encoding used to decode any files read from disk. This includes
    531531template files and initial SQL data files.
    532532
     533FILE_UPLOAD_HANDLERS
     534--------------------
     535
     536**New in Django development version**
     537
     538Default::
     539
     540    ("django.core.files.fileuploadhandler.MemoryFileUploadHandler",
     541     "django.core.files.fileuploadhandler.TemporaryFileUploadHandler",)
     542
     543A tuple of handlers to use for uploading. See `file uploads`_ for details.
     544
     545.. _file uploads: ../upload_handling/
     546
     547FILE_UPLOAD_MAX_MEMORY_SIZE
     548---------------------------
     549
     550**New in Django development version**
     551
     552Default: ``2621440`` (i.e. 2.5 MB).
     553
     554The maximum size (in bytes) that an upload will be before it gets streamed to
     555the file system. See `file uploads`_ for details.
     556
     557FILE_UPLOAD_TEMP_DIR
     558--------------------
     559
     560**New in Django development version**
     561
     562Default: ``None``
     563
     564The directory to store data temporarily while uploading files. If ``None``,
     565Django will use the standard temporary directory for the operating system. For
     566example, this will default to '/tmp' on *nix-style operating systems.
     567
     568See `file uploads`_ for details.
     569
    533570FIXTURE_DIRS
    534571-------------
    535572
  • new file docs/upload_handling.txt

    diff --git a/docs/upload_handling.txt b/docs/upload_handling.txt
    new file mode 100644
    index 0000000..068acf3
    - +  
     1============
     2File Uploads
     3============
     4
     5**New in Django development version**
     6
     7Most Web sites wouldn't be complete without a way to upload files. When Django
     8handles a file upload, the file data ends up placed in ``request.FILES`` (for
     9more on the ``request`` object see the documentation for `request and response
     10objects`_). This document explains how files are stored on disk an in memory,
     11and how to customize the default behavior.
     12
     13.. _request and response objects: ../request_response/#attributes
     14
     15Basic file uploads
     16==================
     17
     18Consider a simple form containing a ``FileField``::
     19
     20    from django import newforms as forms
     21
     22    class UploadFileForm(forms.Form):
     23        title = forms.CharField(max_length=50)
     24        file  = forms.FileField()
     25       
     26A view handling this form will receive the file data in ``request.FILES``, which
     27is a dictionary containing a key for each ``FileField`` (or ``ImageField``, or
     28other ``FileField`` subclass) in the form. So the data from the above form would
     29be accessible as ``request.FILES['file']``.
     30
     31Most of the time, you'll simply pass the file data from ``request`` into the
     32form as described in `binding uploaded files to a form`_. This would look
     33something like::
     34
     35    from django.http import HttpResponseRedirect
     36    from django.shortcuts import render_to_response
     37
     38    # Imaginary function to handle an uploaded file.
     39    from somewhere import handle_uploaded_file
     40
     41    def upload_file(request):
     42        if request.method == 'POST':
     43            form = UploadFileForm(request.POST, request.FILES)
     44            if form.is_valid():
     45                handle_uploaded_file(request.FILES['file'])
     46                return HttpResponseRedirect('/success/url/')
     47        else:
     48            form = UploadFileForm()
     49        return render_to_response('upload.html', {'form': form})
     50
     51.. _binding uploaded files to a form: ../newforms/#binding-uploaded-files-to-a- form
     52
     53Notice that we have to pass ``request.FILES`` into the form's constructor; this
     54is how file data gets bound into a form.
     55
     56Handling uploaded files
     57-----------------------
     58
     59The final piece of the puzzle is handling the actual file data from
     60``request.FILES``. Each entry in this dictionary is an ``UploadedFile`` object
     61-- a simple wrapper around an uploaded file. You'll usually use one of these
     62methods to access the uploaded content:
     63
     64    ``UploadedFile.read()``
     65        Read the entire uploaded data from the file. Be careful with this
     66        method: if the uploaded file is huge it can overwhelm your system if you
     67        try to read it into memory. You'll probably want to use ``chunk()``
     68        instead; see below.
     69       
     70    ``UploadedFile.multiple_chunks()``
     71        Returns ``True`` if the uploaded file is big enough to require
     72        reading in multiple chunks. By default this will be any file
     73        larger than 2.5 megabytes, but that's configurable; see below.
     74   
     75    ``UploadedFile.chunks()``
     76        A generator returning chunks of the file. If ``multiple_chunks()`` is
     77        ``True``, you should use this method in a loop instead of ``read()``.
     78       
     79        In practice, it's often easiest simply to use ``chunks()`` all the time;
     80        see the example below.
     81   
     82    ``UploadedFile.file_name``
     83        The name of the uploaded file (e.g. ``my_file.txt``).
     84       
     85    ``UploadedFile.file_size``
     86        The size, in bytes, of the uploaded file.
     87       
     88There are a few other methods and attributes available on ``UploadedFile``
     89objects; see `UploadedFile objects`_ for a complete reference.
     90
     91Putting it all together, here's a common way you might handle an uploaded file::
     92   
     93    def handle_uploaded_file(f):
     94        destination = open('some/file/name.txt', 'wb')
     95        for chunk in f.chunks():
     96            destination.write(chunk)
     97
     98Looping over ``UploadedFile.chunks()`` instead of using ``read()`` ensures that
     99large files don't overwhelm your system's memory.
     100
     101Where uploaded data is stored
     102-----------------------------
     103
     104Before you save uploaded files, the data needs to be stored somewhere.
     105
     106By default, if an uploaded file is smaller than 2.5 megabytes, Django will hold
     107the entire contents of the upload in memory. This means that saving the file
     108involves only a read from memory and a write to disk and thus is very fast.
     109
     110However, if an uploaded file is too large, Django will write the uploaded file
     111to a temporary file stored in your system's temporary directory. On a Unix-like
     112platform this means you can expect Django to generate a file called something
     113like ``/tmp/tmpzfp6I6.upload``. If an upload is large enough, you can watch this
     114file grow in size as Django streams the data onto disk.
     115
     116These specifics -- 2.5 megabytes; ``/tmp``; etc. -- are simply "reasonable
     117defaults". Read on for details on how you can customize or completely replace
     118upload behavior.
     119
     120Changing upload handler behavior
     121--------------------------------
     122
     123Three `settings`_ control Django's file upload behavior:
     124
     125    ``FILE_UPLOAD_MAX_MEMORY_SIZE``
     126        The maximum size, in bytes, for files that will be uploaded
     127        into memory. Files larger than ``FILE_UPLOAD_MAX_MEMORY_SIZE``
     128        will be streamed to disk.
     129       
     130        Defaults to 2.5 megabytes.
     131       
     132    ``FILE_UPLOAD_TEMP_DIR``
     133        The directory where uploaded files larger than ``FILE_UPLOAD_TEMP_DIR``
     134        will be stored.
     135       
     136        Defaults to your system's standard temporary directory (i.e. ``/tmp`` on
     137        most Unix-like systems).
     138       
     139    ``FILE_UPLOAD_HANDLERS``
     140        The actual handlers for uploaded files. Changing this setting
     141        allows complete customization -- even replacement -- of
     142        Django's upload process. See `upload handlers`_, below,
     143        for details.
     144       
     145        Defaults to::
     146       
     147            ("django.core.files.uploadhandler.MemoryFileUploadHandler",
     148             "django.core.files.uploadhandler.TemporaryFileUploadHandler",)
     149           
     150        Which means "try to upload to memory first, then fall back to temporary
     151        files."
     152
     153.. _settings: ../settings/
     154
     155``UploadedFile`` objects
     156========================
     157
     158All ``UploadedFile`` objects define the following methods/attributes:
     159
     160    ``UploadedFile.read(self, num_bytes=None)``
     161        Returns a byte string of length ``num_bytes``, or the complete file if
     162        ``num_bytes`` is ``None``.
     163
     164    ``UploadedFile.chunk(self, chunk_size=None)``
     165        A generator yielding small chunks from the file. If ``chunk_size`` isn't
     166        given, chunks will be 64 kb.
     167
     168    ``UploadedFile.multiple_chunks(self, chunk_size=None)``
     169        Returns ``True`` if you can expect more than one chunk when calling
     170        ``UploadedFile.chunk(self, chunk_size)``.
     171
     172    ``UploadedFile.file_size``
     173        The size, in bytes, of the uploaded file.
     174   
     175    ``UploadedFile.file_name``
     176        The name of the uploaded file as provided by the user.
     177   
     178    ``UploadedFile.content_type``
     179        The content-type header uploaded with the file (e.g. ``text/plain`` or
     180        ``application/pdf``). Like any data supplied by the user, you shouldn't
     181        trust that the uploaded file is actually this type. You'll still need to
     182        validate that the file contains the content that the content-type header
     183        claims -- "trust but verify."
     184   
     185    ``UploadedFile.charset``
     186        For ``text/*`` content-types, the character set (i.e. ``utf8``) supplied
     187        by the browser. Again, "trust but verify" is the best policy here.
     188
     189    ``UploadedFile.temporary_file_path()``
     190        Only files uploaded onto disk will have this method; it returns the full
     191        path to the temporary uploaded file.
     192
     193Upload Handlers
     194===============
     195
     196When a user uploads a file, Django passes off the file data to an *upload
     197handler* -- a small class that handles file data as it gets uploaded. Upload
     198handlers are initially defined in the ``FILE_UPLOAD_HANDLERS`` setting, which
     199defaults to::
     200
     201    ("django.core.files.uploadhandler.MemoryFileUploadHandler",
     202     "django.core.files.uploadhandler.TemporaryFileUploadHandler",)
     203
     204Together the ``MemoryFileUploadHandler`` and ``TemporaryFileUploadHandler``
     205provide Django's default file upload behavior of reading small files into memory
     206and large ones onto disk.
     207
     208You can write custom handlers that customize how Django handles files. You
     209could, for example, use custom handlers to enforce user-level quotas, compress
     210data on the fly, render progress bars, and even send data to another storage
     211location directly without storing it locally.
     212
     213Modifying upload handlers on the fly
     214------------------------------------
     215
     216Sometimes particular views require different upload behavior. In these cases,
     217you can override upload handlers on a per-request basis by modifying
     218``request.upload_handlers``. By default, this list will contain the upload
     219handlers given by ``FILE_UPLOAD_HANDLERS``, but you can modify the list as you
     220would any other list.
     221
     222For instance, suppose you've written a ``ProgressBarUploadHandler`` that
     223provides feedback on upload progress to some sort of AJAX widget. You'd add this
     224handler to your upload handers like this::
     225
     226    request.upload_handlers.insert(0, ProgressBarUploadHandler())
     227
     228You'd probably want to use ``list.insert()`` in this case (instead of
     229``append()``) because a progress bar handler would need to run *before* any
     230other handlers. Remember, the upload handlers are processed in order.
     231
     232If you want to replace the upload handlers completely, you can just assign a new
     233list::
     234
     235   request.upload_handlers = [ProgressBarUploadHandler()]
     236
     237.. note::
     238
     239    You can only modify upload handlers *before* accessing ``request.FILES`` --
     240    it doesn't make sense to change upload handlers after upload handling has
     241    already started. If you try to modify ``request.upload_handlers`` after
     242    reading from ``request.FILES`` Django will throw an error.
     243
     244    Thus, you should always modify uploading handlers as early in your view as
     245    possible.
     246
     247Writing custom upload handlers
     248------------------------------
     249
     250All file upload handlers should be subclasses of
     251``django.core.files.uploadhandler.FileUploadHandler``. You can define upload
     252handlers wherever you wish.
     253
     254Required methods
     255~~~~~~~~~~~~~~~~
     256
     257Custom file upload handlers **must** define the following methods:
     258
     259    ``FileUploadHandler.receive_data_chunk(self, raw_data, start)``
     260        Receives a "chunk" of data from the file upload.
     261
     262        ``raw_data`` is a byte string containing the uploaded data.
     263
     264        ``start`` is the position in the file where this ``raw_data`` chunk
     265        begins.
     266
     267        The data you return will get fed into the subsequent upload handlers'
     268        ``receive_data_chunk`` methods. In this way, one handler can be a
     269        "filter" for other handlers.
     270
     271        Return ``None`` from ``receive_data_chunk`` to sort-circuit remaining
     272        upload handlers from getting this chunk.. This is useful if you're
     273        storing the uploaded data yourself and don't want future handlers to
     274        store a copy of the data.
     275
     276        If you raise a ``StopUpload`` or a ``SkipFile`` exception, the upload
     277        will abort or the file will be completely skipped.
     278
     279    ``FileUploadHandler.file_complete(self, file_size)``
     280        Called when a file has finished uploading.
     281
     282        The handler should return an ``UploadedFile`` object that will be stored
     283        in ``request.FILES``. Handlers may also return ``None`` to indicate that
     284        the ``UploadedFile`` object should come from subsequent upload handlers.
     285
     286Optional methods
     287~~~~~~~~~~~~~~~~
     288
     289Custom upload handlers may also define any of the following optional methods or
     290attributes:
     291
     292    ``FileUploadHandler.chunk_size``
     293        Size, in bytes, of the "chunks" Django should store into memory and feed
     294        into the handler. That is, this attribute controls the size of chunks
     295        fed into ``FileUploadHandler.receive_data_chunk``.
     296
     297        For maximum performance the chunk sizes should be divisible by ``4`` and
     298        should not exceed 2 GB (2\ :sup:`31` bytes) in size. When there are
     299        multiple chunk sizes provided by multiple handlers, Django will use the
     300        smallest chunk size defined by any handler.
     301
     302        The default is 64*2\ :sup:`10` bytes, or 64 Kb.
     303
     304    ``FileUploadHandler.new_file(self, field_name, file_name, content_type, content_length, charset)``
     305        Callback signaling that a new file upload is starting. This is called
     306        before any data has been fed to any upload handlers.
     307
     308        ``field_name`` is a string name of the file ``<input>`` field.
     309
     310        ``file_name`` is the unicode filename that was provided by the browser.
     311
     312        ``content_type`` is the MIME type provided by the browser -- E.g.
     313        ``'image/jpeg'``.
     314
     315        ``content_length`` is the length of the image given by the browser.
     316        Sometimes this won't be provided and will be ``None``., ``None``
     317        otherwise.
     318
     319        ``charset`` is the character set (i.e. ``utf8``) given by the browser.
     320        Like ``content_length``, this sometimes won't be provided.
     321
     322        This method may raise a ``StopFutureHandlers`` exception to prevent
     323        future handlers from handling this file.
     324
     325    ``FileUploadHandler.upload_complete(self)``
     326        Callback signaling that the entire upload (all files) has completed.
     327
     328    ``FileUploadHandler.``handle_raw_input(self, input_data, META, content_length, boundary, encoding)``
     329        Allows the handler to completely override the parsing of the raw
     330        HTTP input.
     331
     332        ``input_data`` is a file-like object that supports ``read()``-ing.
     333
     334        ``META`` is the same object as ``request.META``.
     335
     336        ``content_length`` is the length of the data in ``input_data``. Don't
     337        read more than ``content_length`` bytes from ``input_data``.
     338
     339        ``boundary`` is the MIME boundary for this request.
     340
     341        ``encoding`` is the encoding of the request.
     342
     343        Return ``None`` if you want upload handling to continue, or a tuple of
     344        ``(POST, FILES)`` if you want to return the new data structures suitable
     345        for the request directly.
     346
  • tests/modeltests/model_forms/models.py

    diff --git a/tests/modeltests/model_forms/models.py b/tests/modeltests/model_forms/models.py
    index 470312f..c856720 100644
    a b class TextFile(models.Model): 
    6767
    6868class ImageFile(models.Model):
    6969    description = models.CharField(max_length=20)
    70     image = models.FileField(upload_to=tempfile.gettempdir())
     70    try:
     71        # If PIL is available, try testing PIL.
     72        # Otherwise, it's equivalent to TextFile above.
     73        import Image
     74        image = models.ImageField(upload_to=tempfile.gettempdir())
     75    except ImportError:
     76        image = models.FileField(upload_to=tempfile.gettempdir())
    7177
    7278    def __unicode__(self):
    7379        return self.description
    class ImageFile(models.Model): 
    7581__test__ = {'API_TESTS': """
    7682>>> from django import newforms as forms
    7783>>> from django.newforms.models import ModelForm
     84>>> from django.core.files.uploadedfile import SimpleUploadedFile
    7885
    7986The bare bones, absolutely nothing custom, basic case.
    8087
    False 
    792799
    793800# Upload a file and ensure it all works as expected.
    794801
    795 >>> f = TextFileForm(data={'description': u'Assistance'}, files={'file': {'filename': 'test1.txt', 'content': 'hello world'}})
     802>>> f = TextFileForm(data={'description': u'Assistance'}, files={'file': SimpleUploadedFile('test1.txt', 'hello world')})
     803>>> f.is_valid()
     804True
     805>>> type(f.cleaned_data['file'])
     806<class 'django.newforms.fields.UploadedFile'>
     807>>> instance = f.save()
     808>>> instance.file
     809u'...test1.txt'
     810
     811>>> os.unlink(instance.get_file_filename())
     812
     813>>> f = TextFileForm(data={'description': u'Assistance'}, files={'file': SimpleUploadedFile('test1.txt', 'hello world')})
    796814>>> f.is_valid()
    797815True
    798816>>> type(f.cleaned_data['file'])
    u'...test1.txt' 
    814832u'...test1.txt'
    815833
    816834# Delete the current file since this is not done by Django.
    817 
    818835>>> os.unlink(instance.get_file_filename())
    819836
    820837# Override the file by uploading a new one.
    821838
    822 >>> f = TextFileForm(data={'description': u'Assistance'}, files={'file': {'filename': 'test2.txt', 'content': 'hello world'}}, instance=instance)
     839>>> f = TextFileForm(data={'description': u'Assistance'}, files={'file': SimpleUploadedFile('test2.txt', 'hello world')}, instance=instance)
    823840>>> f.is_valid()
    824841True
    825842>>> instance = f.save()
    826843>>> instance.file
    827844u'...test2.txt'
    828845
     846# Delete the current file since this is not done by Django.
     847>>> os.unlink(instance.get_file_filename())
     848
     849>>> f = TextFileForm(data={'description': u'Assistance'}, files={'file': SimpleUploadedFile('test2.txt', 'hello world')})
     850>>> f.is_valid()
     851True
     852>>> instance = f.save()
     853>>> instance.file
     854u'...test2.txt'
     855
     856# Delete the current file since this is not done by Django.
     857>>> os.unlink(instance.get_file_filename())
     858
    829859>>> instance.delete()
    830860
    831861# Test the non-required FileField
    True 
    838868>>> instance.file
    839869''
    840870
    841 >>> f = TextFileForm(data={'description': u'Assistance'}, files={'file': {'filename': 'test3.txt', 'content': 'hello world'}}, instance=instance)
     871>>> f = TextFileForm(data={'description': u'Assistance'}, files={'file': SimpleUploadedFile('test3.txt', 'hello world')}, instance=instance)
    842872>>> f.is_valid()
    843873True
    844874>>> instance = f.save()
    845875>>> instance.file
    846876u'...test3.txt'
     877
     878# Delete the current file since this is not done by Django.
     879>>> os.unlink(instance.get_file_filename())
     880>>> instance.delete()
     881
     882>>> f = TextFileForm(data={'description': u'Assistance'}, files={'file': SimpleUploadedFile('test3.txt', 'hello world')})
     883>>> f.is_valid()
     884True
     885>>> instance = f.save()
     886>>> instance.file
     887u'...test3.txt'
     888
     889# Delete the current file since this is not done by Django.
     890>>> os.unlink(instance.get_file_filename())
    847891>>> instance.delete()
    848892
    849893# ImageField ###################################################################
    u'...test3.txt' 
    858902
    859903>>> image_data = open(os.path.join(os.path.dirname(__file__), "test.png")).read()
    860904
    861 >>> f = ImageFileForm(data={'description': u'An image'}, files={'image': {'filename': 'test.png', 'content': image_data}})
     905>>> f = ImageFileForm(data={'description': u'An image'}, files={'image': SimpleUploadedFile('test.png', image_data)})
     906>>> f.is_valid()
     907True
     908>>> type(f.cleaned_data['image'])
     909<class 'django.newforms.fields.UploadedFile'>
     910>>> instance = f.save()
     911>>> instance.image
     912u'...test.png'
     913
     914# Delete the current file since this is not done by Django.
     915>>> os.unlink(instance.get_image_filename())
     916
     917>>> f = ImageFileForm(data={'description': u'An image'}, files={'image': SimpleUploadedFile('test.png', image_data)})
    862918>>> f.is_valid()
    863919True
    864920>>> type(f.cleaned_data['image'])
    u'...test.png' 
    885941
    886942# Override the file by uploading a new one.
    887943
    888 >>> f = ImageFileForm(data={'description': u'Changed it'}, files={'image': {'filename': 'test2.png', 'content': image_data}}, instance=instance)
     944>>> f = ImageFileForm(data={'description': u'Changed it'}, files={'image': SimpleUploadedFile('test2.png', image_data)}, instance=instance)
     945>>> f.is_valid()
     946True
     947>>> instance = f.save()
     948>>> instance.image
     949u'...test2.png'
     950
     951# Delete the current file since this is not done by Django.
     952>>> os.unlink(instance.get_image_filename())
     953>>> instance.delete()
     954
     955>>> f = ImageFileForm(data={'description': u'Changed it'}, files={'image': SimpleUploadedFile('test2.png', image_data)})
    889956>>> f.is_valid()
    890957True
    891958>>> instance = f.save()
    892959>>> instance.image
    893960u'...test2.png'
    894961
     962# Delete the current file since this is not done by Django.
     963>>> os.unlink(instance.get_image_filename())
    895964>>> instance.delete()
    896965
    897966# Test the non-required ImageField
    True 
    904973>>> instance.image
    905974''
    906975
    907 >>> f = ImageFileForm(data={'description': u'And a final one'}, files={'image': {'filename': 'test3.png', 'content': image_data}}, instance=instance)
     976>>> f = ImageFileForm(data={'description': u'And a final one'}, files={'image': SimpleUploadedFile('test3.png', image_data)}, instance=instance)
     977>>> f.is_valid()
     978True
     979>>> instance = f.save()
     980>>> instance.image
     981u'...test3.png'
     982
     983# Delete the current file since this is not done by Django.
     984>>> os.unlink(instance.get_image_filename())
     985>>> instance.delete()
     986
     987>>> f = ImageFileForm(data={'description': u'And a final one'}, files={'image': SimpleUploadedFile('test3.png', image_data)})
    908988>>> f.is_valid()
    909989True
    910990>>> instance = f.save()
  • tests/regressiontests/bug639/tests.py

    diff --git a/tests/regressiontests/bug639/tests.py b/tests/regressiontests/bug639/tests.py
    index f9596d0..2726dec 100644
    a b import unittest 
    99from regressiontests.bug639.models import Photo
    1010from django.http import QueryDict
    1111from django.utils.datastructures import MultiValueDict
     12from django.core.files.uploadedfile import SimpleUploadedFile
    1213
    1314class Bug639Test(unittest.TestCase):
    1415       
    class Bug639Test(unittest.TestCase): 
    2122       
    2223        # Fake a request query dict with the file
    2324        qd = QueryDict("title=Testing&image=", mutable=True)
    24         qd["image_file"] = {
    25             "filename" : "test.jpg",
    26             "content-type" : "image/jpeg",
    27             "content" : img
    28         }
    29        
     25        qd["image_file"] = SimpleUploadedFile('test.jpg', img, 'image/jpeg')
     26
    3027        manip = Photo.AddManipulator()
    3128        manip.do_html2python(qd)
    3229        p = manip.save(qd)
    class Bug639Test(unittest.TestCase): 
    3936        Make sure to delete the "uploaded" file to avoid clogging /tmp.
    4037        """
    4138        p = Photo.objects.get()
    42         os.unlink(p.get_image_filename())
    43  No newline at end of file
     39        os.unlink(p.get_image_filename())
  • tests/regressiontests/datastructures/tests.py

    diff --git a/tests/regressiontests/datastructures/tests.py b/tests/regressiontests/datastructures/tests.py
    index d6141b0..62c57bc 100644
    a b Init from sequence of tuples 
    117117>>> d['person']['2']['firstname']
    118118['Adrian']
    119119
    120 ### FileDict ################################################################
    121 
    122 >>> d = FileDict({'content': 'once upon a time...'})
    123 >>> repr(d)
    124 "{'content': '<omitted>'}"
    125 >>> d = FileDict({'other-key': 'once upon a time...'})
     120### ImmutableList ################################################################
     121>>> d = ImmutableList(range(10))
     122>>> d.sort()
     123Traceback (most recent call last):
     124  File "<stdin>", line 1, in <module>
     125  File "/var/lib/python-support/python2.5/django/utils/datastructures.py", line 359, in complain
     126    raise AttributeError, self.warning
     127AttributeError: ImmutableList object is immutable.
    126128>>> repr(d)
    127 "{'other-key': 'once upon a time...'}"
     129'(0, 1, 2, 3, 4, 5, 6, 7, 8, 9)'
     130>>> d = ImmutableList(range(10), warning="Object is immutable!")
     131>>> d[1]
     1321
     133>>> d[1] = 'test'
     134Traceback (most recent call last):
     135  File "<stdin>", line 1, in <module>
     136  File "/var/lib/python-support/python2.5/django/utils/datastructures.py", line 359, in complain
     137    raise AttributeError, self.warning
     138AttributeError: Object is immutable!
    128139
    129140### DictWrapper #############################################################
    130141
  • new file tests/regressiontests/file_uploads/models.py

    diff --git a/tests/regressiontests/file_uploads/__init__.py b/tests/regressiontests/file_uploads/__init__.py
    new file mode 100644
    index 0000000..e69de29
    diff --git a/tests/regressiontests/file_uploads/models.py b/tests/regressiontests/file_uploads/models.py
    new file mode 100644
    index 0000000..2d5607b
    - +  
     1# This file unintentionally left blank.
     2# Oops.
     3 No newline at end of file
  • new file tests/regressiontests/file_uploads/tests.py

    diff --git a/tests/regressiontests/file_uploads/tests.py b/tests/regressiontests/file_uploads/tests.py
    new file mode 100644
    index 0000000..8992298
    - +  
     1import os
     2import sha
     3import tempfile
     4from django.test import TestCase, client
     5from django.utils import simplejson
     6
     7class FileUploadTests(TestCase):
     8    def test_simple_upload(self):
     9        post_data = {
     10            'name': 'Ringo',
     11            'file_field': open(__file__),
     12        }
     13        response = self.client.post('/file_uploads/upload/', post_data)
     14        self.assertEqual(response.status_code, 200)
     15
     16    def test_large_upload(self):
     17        tdir = tempfile.gettempdir()
     18       
     19        file1 = tempfile.NamedTemporaryFile(suffix=".file1", dir=tdir)
     20        file1.write('a' * (2 ** 21))
     21        file1.seek(0)
     22
     23        file2 = tempfile.NamedTemporaryFile(suffix=".file2", dir=tdir)
     24        file2.write('a' * (10 * 2 ** 20))
     25        file2.seek(0)
     26
     27        # This file contains chinese symbols for a name.
     28        file3 = open(os.path.join(tdir, u'test_&#20013;&#25991;_Orl\u00e9ans.jpg'), 'w+b')
     29        file3.write('b' * (2 ** 10))
     30        file3.seek(0)
     31
     32        post_data = {
     33            'name': 'Ringo',
     34            'file_field1': open(file1.name),
     35            'file_field2': open(file2.name),
     36            'file_unicode': file3,
     37            }
     38
     39        for key in post_data.keys():
     40            try:
     41                post_data[key + '_hash'] = sha.new(post_data[key].read()).hexdigest()
     42                post_data[key].seek(0)
     43            except AttributeError:
     44                post_data[key + '_hash'] = sha.new(post_data[key]).hexdigest()
     45
     46        response = self.client.post('/file_uploads/verify/', post_data)
     47
     48        try:
     49            os.unlink(file3.name)
     50        except:
     51            pass
     52
     53        self.assertEqual(response.status_code, 200)
     54   
     55    def test_dangerous_file_names(self):
     56        """Uploaded file names should be sanitized before ever reaching the view."""
     57        # This test simulates possible directory traversal attacks by a
     58        # malicious uploader We have to do some monkeybusiness here to construct
     59        # a malicious payload with an invalid file name (containing os.sep or
     60        # os.pardir). This similar to what an attacker would need to do when
     61        # trying such an attack.
     62        scary_file_names = [
     63            "/tmp/hax0rd.txt",          # Absolute path, *nix-style.
     64            "C:\\Windows\\hax0rd.txt",  # Absolute path, win-syle.
     65            "C:/Windows/hax0rd.txt",    # Absolute path, broken-style.
     66            "\\tmp\\hax0rd.txt",        # Absolute path, broken in a different way.
     67            "/tmp\\hax0rd.txt",         # Absolute path, broken by mixing.
     68            "subdir/hax0rd.txt",        # Descendant path, *nix-style.
     69            "subdir\\hax0rd.txt",       # Descendant path, win-style.
     70            "sub/dir\\hax0rd.txt",      # Descendant path, mixed.
     71            "../../hax0rd.txt",         # Relative path, *nix-style.
     72            "..\\..\\hax0rd.txt",       # Relative path, win-style.
     73            "../..\\hax0rd.txt"         # Relative path, mixed.
     74        ]
     75       
     76        payload = []
     77        for i, name in enumerate(scary_file_names):
     78            payload.extend([
     79                '--' + client.BOUNDARY,
     80                'Content-Disposition: form-data; name="file%s"; filename="%s"' % (i, name),
     81                'Content-Type: application/octet-stream',
     82                '',
     83                'You got pwnd.'
     84            ])
     85        payload.extend([
     86            '--' + client.BOUNDARY + '--',
     87            '',
     88        ])
     89       
     90        payload = "\r\n".join(payload)
     91        r = {
     92            'CONTENT_LENGTH': len(payload),
     93            'CONTENT_TYPE':   client.MULTIPART_CONTENT,
     94            'PATH_INFO':      "/file_uploads/echo/",
     95            'REQUEST_METHOD': 'POST',
     96            'wsgi.input':     client.FakePayload(payload),
     97        }
     98        response = self.client.request(**r)
     99
     100        # The filenames should have been sanitized by the time it got to the view.
     101        recieved = simplejson.loads(response.content)
     102        for i, name in enumerate(scary_file_names):
     103            got = recieved["file%s" % i]
     104            self.assertEqual(got, "hax0rd.txt")
     105           
     106    def test_filename_overflow(self):
     107        """File names over 256 characters (dangerous on some platforms) get fixed up."""
     108        name = "%s.txt" % ("f"*500)
     109        payload = "\r\n".join([
     110            '--' + client.BOUNDARY,
     111            'Content-Disposition: form-data; name="file"; filename="%s"' % name,
     112            'Content-Type: application/octet-stream',
     113            '',
     114            'Oops.'
     115            '--' + client.BOUNDARY + '--',
     116            '',
     117        ])
     118        r = {
     119            'CONTENT_LENGTH': len(payload),
     120            'CONTENT_TYPE':   client.MULTIPART_CONTENT,
     121            'PATH_INFO':      "/file_uploads/echo/",
     122            'REQUEST_METHOD': 'POST',
     123            'wsgi.input':     client.FakePayload(payload),
     124        }
     125        got = simplejson.loads(self.client.request(**r).content)
     126        self.assert_(len(got['file']) < 256, "Got a long file name (%s characters)." % len(got['file']))
     127       
     128    def test_custom_upload_handler(self):
     129        # A small file (under the 5M quota)               
     130        smallfile = tempfile.NamedTemporaryFile()
     131        smallfile.write('a' * (2 ** 21))
     132
     133        # A big file (over the quota)
     134        bigfile = tempfile.NamedTemporaryFile()
     135        bigfile.write('a' * (10 * 2 ** 20))
     136               
     137        # Small file posting should work.
     138        response = self.client.post('/file_uploads/quota/', {'f': open(smallfile.name)})
     139        got = simplejson.loads(response.content)
     140        self.assert_('f' in got)
     141       
     142        # Large files don't go through.
     143        response = self.client.post("/file_uploads/quota/", {'f': open(bigfile.name)})
     144        got = simplejson.loads(response.content)
     145        self.assert_('f' not in got)
     146       
     147    def test_broken_custom_upload_handler(self):
     148        f = tempfile.NamedTemporaryFile()
     149        f.write('a' * (2 ** 21))
     150       
     151        # AttributeError: You cannot alter upload handlers after the upload has been processed.
     152        self.assertRaises(
     153            AttributeError,
     154            self.client.post,
     155            '/file_uploads/quota/broken/',
     156            {'f': open(f.name)}
     157        )       
     158       
     159 No newline at end of file
  • new file tests/regressiontests/file_uploads/uploadhandler.py

    diff --git a/tests/regressiontests/file_uploads/uploadhandler.py b/tests/regressiontests/file_uploads/uploadhandler.py
    new file mode 100644
    index 0000000..54f82f6
    - +  
     1"""
     2Upload handlers to test the upload API.
     3"""
     4
     5from django.core.files.uploadhandler import FileUploadHandler, StopUpload
     6
     7class QuotaUploadHandler(FileUploadHandler):
     8    """
     9    This test upload handler terminates the connection if more than a quota
     10    (5MB) is uploaded.
     11    """
     12   
     13    QUOTA = 5 * 2**20 # 5 MB
     14   
     15    def __init__(self, request=None):
     16        super(QuotaUploadHandler, self).__init__(request)
     17        self.total_upload = 0
     18       
     19    def receive_data_chunk(self, raw_data, start):
     20        self.total_upload += len(raw_data)
     21        if self.total_upload >= self.QUOTA:
     22            raise StopUpload(connection_reset=True)
     23        return raw_data
     24           
     25    def file_complete(self, file_size):
     26        return None
     27 No newline at end of file
  • new file tests/regressiontests/file_uploads/urls.py

    diff --git a/tests/regressiontests/file_uploads/urls.py b/tests/regressiontests/file_uploads/urls.py
    new file mode 100644
    index 0000000..529bee3
    - +  
     1from django.conf.urls.defaults import *
     2import views
     3
     4urlpatterns = patterns('',
     5    (r'^upload/$',          views.file_upload_view),
     6    (r'^verify/$',          views.file_upload_view_verify),
     7    (r'^echo/$',            views.file_upload_echo),
     8    (r'^quota/$',           views.file_upload_quota),
     9    (r'^quota/broken/$',    views.file_upload_quota_broken),
     10)
  • new file tests/regressiontests/file_uploads/views.py

    diff --git a/tests/regressiontests/file_uploads/views.py b/tests/regressiontests/file_uploads/views.py
    new file mode 100644
    index 0000000..833cf90
    - +  
     1import os
     2import sha
     3from django.core.files.uploadedfile import UploadedFile
     4from django.http import HttpResponse, HttpResponseServerError
     5from django.utils import simplejson
     6from uploadhandler import QuotaUploadHandler
     7
     8def file_upload_view(request):
     9    """
     10    Check that a file upload can be updated into the POST dictionary without
     11    going pear-shaped.
     12    """
     13    form_data = request.POST.copy()
     14    form_data.update(request.FILES)
     15    if isinstance(form_data.get('file_field'), UploadedFile) and isinstance(form_data['name'], unicode):
     16        # If a file is posted, the dummy client should only post the file name,
     17        # not the full path.
     18        if os.path.dirname(form_data['file_field'].file_name) != '':
     19            return HttpResponseServerError()           
     20        return HttpResponse('')
     21    else:
     22        return HttpResponseServerError()
     23
     24def file_upload_view_verify(request):
     25    """
     26    Use the sha digest hash to verify the uploaded contents.
     27    """
     28    form_data = request.POST.copy()
     29    form_data.update(request.FILES)
     30
     31    # Check to see if unicode names worked out.
     32    if not request.FILES['file_unicode'].file_name.endswith(u'test_\u4e2d\u6587_Orl\xe9ans.jpg'):
     33        return HttpResponseServerError()
     34
     35    for key, value in form_data.items():
     36        if key.endswith('_hash'):
     37            continue
     38        if key + '_hash' not in form_data:
     39            continue
     40        submitted_hash = form_data[key + '_hash']
     41        if isinstance(value, UploadedFile):
     42            new_hash = sha.new(value.read()).hexdigest()
     43        else:
     44            new_hash = sha.new(value).hexdigest()
     45        if new_hash != submitted_hash:
     46            return HttpResponseServerError()
     47
     48    return HttpResponse('')
     49
     50def file_upload_echo(request):
     51    """
     52    Simple view to echo back info about uploaded files for tests.
     53    """
     54    r = dict([(k, f.file_name) for k, f in request.FILES.items()])
     55    return HttpResponse(simplejson.dumps(r))
     56   
     57def file_upload_quota(request):
     58    """
     59    Dynamically add in an upload handler.
     60    """
     61    request.upload_handlers.insert(0, QuotaUploadHandler())
     62    return file_upload_echo(request)
     63       
     64def file_upload_quota_broken(request):
     65    """
     66    You can't change handlers after reading FILES; this view shouldn't work.
     67    """
     68    response = file_upload_echo(request)
     69    request.upload_handlers.insert(0, QuotaUploadHandler())
     70    return response
     71 No newline at end of file
  • tests/regressiontests/forms/error_messages.py

    diff --git a/tests/regressiontests/forms/error_messages.py b/tests/regressiontests/forms/error_messages.py
    index 9f972f5..c7b301b 100644
    a b  
    11# -*- coding: utf-8 -*-
    22tests = r"""
    33>>> from django.newforms import *
     4>>> from django.core.files.uploadedfile import SimpleUploadedFile
    45
    56# CharField ###################################################################
    67
    ValidationError: [u'REQUIRED'] 
    214215Traceback (most recent call last):
    215216...
    216217ValidationError: [u'INVALID']
    217 >>> f.clean({})
     218>>> f.clean(SimpleUploadedFile('name', None))
    218219Traceback (most recent call last):
    219220...
    220 ValidationError: [u'MISSING']
    221 >>> f.clean({'filename': 'name', 'content':''})
     221ValidationError: [u'EMPTY FILE']
     222>>> f.clean(SimpleUploadedFile('name', ''))
    222223Traceback (most recent call last):
    223224...
    224225ValidationError: [u'EMPTY FILE']
  • tests/regressiontests/forms/fields.py

    diff --git a/tests/regressiontests/forms/fields.py b/tests/regressiontests/forms/fields.py
    index f3b6a96..8fe4bc1 100644
    a b  
    22tests = r"""
    33>>> from django.newforms import *
    44>>> from django.newforms.widgets import RadioFieldRenderer
     5>>> from django.core.files.uploadedfile import SimpleUploadedFile
    56>>> import datetime
    67>>> import time
    78>>> import re
    ValidationError: [u'This field is required.'] 
    770771>>> f.clean(None, 'files/test2.pdf')
    771772'files/test2.pdf'
    772773
    773 >>> f.clean({})
     774>>> f.clean(SimpleUploadedFile('', ''))
    774775Traceback (most recent call last):
    775776...
    776 ValidationError: [u'No file was submitted.']
     777ValidationError: [u'No file was submitted. Check the encoding type on the form.']
    777778
    778 >>> f.clean({}, '')
     779>>> f.clean(SimpleUploadedFile('', ''), '')
    779780Traceback (most recent call last):
    780781...
    781 ValidationError: [u'No file was submitted.']
     782ValidationError: [u'No file was submitted. Check the encoding type on the form.']
    782783
    783 >>> f.clean({}, 'files/test3.pdf')
     784>>> f.clean(None, 'files/test3.pdf')
    784785'files/test3.pdf'
    785786
    786787>>> f.clean('some content that is not a file')
    Traceback (most recent call last): 
    788789...
    789790ValidationError: [u'No file was submitted. Check the encoding type on the form.']
    790791
    791 >>> f.clean({'filename': 'name', 'content': None})
     792>>> f.clean(SimpleUploadedFile('name', None))
    792793Traceback (most recent call last):
    793794...
    794795ValidationError: [u'The submitted file is empty.']
    795796
    796 >>> f.clean({'filename': 'name', 'content': ''})
     797>>> f.clean(SimpleUploadedFile('name', ''))
    797798Traceback (most recent call last):
    798799...
    799800ValidationError: [u'The submitted file is empty.']
    800801
    801 >>> type(f.clean({'filename': 'name', 'content': 'Some File Content'}))
     802>>> type(f.clean(SimpleUploadedFile('name', 'Some File Content')))
    802803<class 'django.newforms.fields.UploadedFile'>
    803804
    804 >>> type(f.clean({'filename': 'name', 'content': 'Some File Content'}, 'files/test4.pdf'))
     805>>> type(f.clean(SimpleUploadedFile('name', 'Some File Content'), 'files/test4.pdf'))
    805806<class 'django.newforms.fields.UploadedFile'>
    806807
    807808# URLField ##################################################################
  • tests/regressiontests/forms/forms.py

    diff --git a/tests/regressiontests/forms/forms.py b/tests/regressiontests/forms/forms.py
    index 7fc206d..041fa40 100644
    a b  
    11# -*- coding: utf-8 -*-
    22tests = r"""
    33>>> from django.newforms import *
     4>>> from django.core.files.uploadedfile import SimpleUploadedFile
    45>>> import datetime
    56>>> import time
    67>>> import re
    not request.POST. 
    14651466>>> print f
    14661467<tr><th>File1:</th><td><ul class="errorlist"><li>This field is required.</li></ul><input type="file" name="file1" /></td></tr>
    14671468
    1468 >>> f = FileForm(data={}, files={'file1': {'filename': 'name', 'content':''}}, auto_id=False)
     1469>>> f = FileForm(data={}, files={'file1': SimpleUploadedFile('name', '')}, auto_id=False)
    14691470>>> print f
    14701471<tr><th>File1:</th><td><ul class="errorlist"><li>The submitted file is empty.</li></ul><input type="file" name="file1" /></td></tr>
    14711472
    not request.POST. 
    14731474>>> print f
    14741475<tr><th>File1:</th><td><ul class="errorlist"><li>No file was submitted. Check the encoding type on the form.</li></ul><input type="file" name="file1" /></td></tr>
    14751476
    1476 >>> f = FileForm(data={}, files={'file1': {'filename': 'name', 'content':'some content'}}, auto_id=False)
     1477>>> f = FileForm(data={}, files={'file1': SimpleUploadedFile('name', 'some content')}, auto_id=False)
    14771478>>> print f
    14781479<tr><th>File1:</th><td><input type="file" name="file1" /></td></tr>
    14791480>>> f.is_valid()
  • tests/regressiontests/test_client_regress/models.py

    diff --git a/tests/regressiontests/test_client_regress/models.py b/tests/regressiontests/test_client_regress/models.py
    index a204ec3..37b997b 100644
    a b from django.test import Client, TestCase 
    66from django.core.urlresolvers import reverse
    77from django.core.exceptions import SuspiciousOperation
    88import os
     9import sha
    910
    1011class AssertContainsTests(TestCase):
    1112    def test_contains(self):
    class AssertFormErrorTests(TestCase): 
    240241        except AssertionError, e:
    241242            self.assertEqual(str(e), "The form 'form' in context 0 does not contain the non-field error 'Some error.' (actual errors: )")
    242243
    243 class FileUploadTests(TestCase):
    244     def test_simple_upload(self):
    245         fd = open(os.path.join(os.path.dirname(__file__), "views.py"))
    246         post_data = {
    247             'name': 'Ringo',
    248             'file_field': fd,
    249         }
    250         response = self.client.post('/test_client_regress/file_upload/', post_data)
    251         self.assertEqual(response.status_code, 200)
    252 
    253244class LoginTests(TestCase):
    254245    fixtures = ['testdata']
    255246
    class LoginTests(TestCase): 
    269260        # default client.
    270261        self.assertRedirects(response, "http://testserver/test_client_regress/get_view/")
    271262
    272 
    273263class URLEscapingTests(TestCase):
    274264    def test_simple_argument_get(self):
    275265        "Get a view that has a simple string argument"
  • tests/regressiontests/test_client_regress/urls.py

    diff --git a/tests/regressiontests/test_client_regress/urls.py b/tests/regressiontests/test_client_regress/urls.py
    index dc26d12..12f6afa 100644
    a b import views 
    33
    44urlpatterns = patterns('',
    55    (r'^no_template_view/$', views.no_template_view),
    6     (r'^file_upload/$', views.file_upload_view),
    76    (r'^staff_only/$', views.staff_only_view),
    87    (r'^get_view/$', views.get_view),
    98    url(r'^arg_view/(?P<name>.+)/$', views.view_with_argument, name='arg_view'),
  • tests/regressiontests/test_client_regress/views.py

    diff --git a/tests/regressiontests/test_client_regress/views.py b/tests/regressiontests/test_client_regress/views.py
    index 9632c17..d703c82 100644
    a b  
    1 import os
    2 
    31from django.contrib.auth.decorators import login_required
    4 from django.http import HttpResponse, HttpResponseRedirect, HttpResponseServerError
     2from django.http import HttpResponse, HttpResponseRedirect
    53from django.core.exceptions import SuspiciousOperation
    64
    75def no_template_view(request):
    86    "A simple view that expects a GET request, and returns a rendered template"
    97    return HttpResponse("No template used. Sample content: twice once twice. Content ends.")
    108
    11 def file_upload_view(request):
    12     """
    13     Check that a file upload can be updated into the POST dictionary without
    14     going pear-shaped.
    15     """
    16     form_data = request.POST.copy()
    17     form_data.update(request.FILES)
    18     if isinstance(form_data['file_field'], dict) and isinstance(form_data['name'], unicode):
    19         # If a file is posted, the dummy client should only post the file name,
    20         # not the full path.
    21         if os.path.dirname(form_data['file_field']['filename']) != '':
    22             return HttpResponseServerError()           
    23         return HttpResponse('')
    24     else:
    25         return HttpResponseServerError()
    26 
    279def staff_only_view(request):
    2810    "A view that can only be visited by staff. Non staff members get an exception"
    2911    if request.user.is_staff:
    3012        return HttpResponse('')
    3113    else:
    3214        raise SuspiciousOperation()
    33    
     15
    3416def get_view(request):
    3517    "A simple login protected view"
    3618    return HttpResponse("Hello world")
    def view_with_argument(request, name): 
    5133def login_protected_redirect_view(request):
    5234    "A view that redirects all requests to the GET view"
    5335    return HttpResponseRedirect('/test_client_regress/get_view/')
    54 login_protected_redirect_view = login_required(login_protected_redirect_view)
    55  No newline at end of file
     36login_protected_redirect_view = login_required(login_protected_redirect_view)
  • tests/urls.py

    diff --git a/tests/urls.py b/tests/urls.py
    index dbdf9a8..cea453e 100644
    a b urlpatterns = patterns('', 
    55    (r'^test_client/', include('modeltests.test_client.urls')),
    66    (r'^test_client_regress/', include('regressiontests.test_client_regress.urls')),
    77
     8    # File upload test views
     9    (r'^file_uploads/', include('regressiontests.file_uploads.urls')),
     10
    811    # Always provide the auth system login and logout views
    912    (r'^accounts/login/$', 'django.contrib.auth.views.login', {'template_name': 'login.html'}),
    1013    (r'^accounts/logout/$', 'django.contrib.auth.views.logout'),
Back to Top