Django

Code

Changeset 7859

Show
Ignore:
Timestamp:
07/07/08 18:16:00 (2 months ago)
Author:
jacob
Message:

Fixed #7614: the quickening has come, and there now is only one UploadedFile?. On top of that, UploadedFile?'s interface has been improved:

  • The API now more closely matches a proper file API. This unfortunately means a few backwards-incompatible renamings; see BackwardsIncompatibleChanges. This refs #7593.
  • While we were at it, renamed chunk() to chunks() to clarify that it's an iterator.
  • Temporary uploaded files now property use the tempfile library behind the scenes which should ensure better cleanup of tempfiles (refs #7593 again).

Thanks to Mike Axiak for the bulk of this patch.

Files:

Legend:

Unmodified
Added
Removed
Modified
Copied
Moved
  • django/trunk/django/core/files/uploadedfile.py

    r7829 r7859  
    44 
    55import os 
     6import tempfile 
     7import warnings 
    68try: 
    79    from cStringIO import StringIO 
     
    911    from StringIO import StringIO 
    1012 
    11 __all__ = ('UploadedFile', 'TemporaryUploadedFile', 'InMemoryUploadedFile') 
     13from django.conf import settings 
     14 
     15__all__ = ('UploadedFile', 'TemporaryUploadedFile', 'InMemoryUploadedFile', 'SimpleUploadedFile') 
     16 
     17# Because we fooled around with it a bunch, UploadedFile has a bunch 
     18# of deprecated properties. This little shortcut helps define 'em 
     19# without too much code duplication. 
     20def deprecated_property(old, new, readonly=False): 
     21    def issue_warning(): 
     22        warnings.warn( 
     23            message = "UploadedFile.%s is deprecated; use UploadedFile.%s instead." % (old, new), 
     24            category = DeprecationWarning, 
     25            stacklevel = 3 
     26        ) 
     27     
     28    def getter(self): 
     29        issue_warning() 
     30        return getattr(self, new) 
     31         
     32    def setter(self, value): 
     33        issue_warning() 
     34        setattr(self, new, value) 
     35         
     36    if readonly: 
     37        return property(getter) 
     38    else: 
     39        return property(getter, setter) 
    1240 
    1341class UploadedFile(object): 
     
    2149    DEFAULT_CHUNK_SIZE = 64 * 2**10 
    2250 
    23     def __init__(self, file_name=None, content_type=None, file_size=None, charset=None): 
    24         self.file_name = file_name 
    25         self.file_size = file_size 
     51    def __init__(self, name=None, content_type=None, size=None, charset=None): 
     52        self.name = name 
     53        self.size = size 
    2654        self.content_type = content_type 
    2755        self.charset = charset 
    2856 
    2957    def __repr__(self): 
    30         return "<%s: %s (%s)>" % (self.__class__.__name__, self.file_name, self.content_type) 
    31  
    32     def _set_file_name(self, name): 
     58        return "<%s: %s (%s)>" % (self.__class__.__name__, self.name, self.content_type) 
     59 
     60    def _get_name(self): 
     61        return self._name 
     62 
     63    def _set_name(self, name): 
    3364        # Sanitize the file name so that it can't be dangerous. 
    3465        if name is not None: 
    3566            # Just use the basename of the file -- anything else is dangerous. 
    3667            name = os.path.basename(name) 
    37              
     68 
    3869            # File names longer than 255 characters can cause problems on older OSes. 
    3970            if len(name) > 255: 
    4071                name, ext = os.path.splitext(name) 
    4172                name = name[:255 - len(ext)] + ext 
    42                  
    43         self._file_name = name 
    44          
    45     def _get_file_name(self): 
    46         return self._file_name 
    47          
    48     file_name = property(_get_file_name, _set_file_name) 
    49  
    50     def chunk(self, chunk_size=None): 
     73 
     74        self._name = name 
     75 
     76    name = property(_get_name, _set_name) 
     77 
     78    def chunks(self, chunk_size=None): 
    5179        """ 
    5280        Read the file and yield chucks of ``chunk_size`` bytes (defaults to 
     
    5987            self.seek(0) 
    6088        # Assume the pointer is at zero... 
    61         counter = self.file_size 
     89        counter = self.size 
    6290 
    6391        while counter > 0: 
     
    6593            counter -= chunk_size 
    6694 
     95    # Deprecated properties 
     96    file_name = deprecated_property(old="file_name", new="name") 
     97    file_size = deprecated_property(old="file_size", new="size") 
     98    data = deprecated_property(old="data", new="read", readonly=True) 
     99    chunk = deprecated_property(old="chunk", new="chunks", readonly=True) 
     100 
    67101    def multiple_chunks(self, chunk_size=None): 
    68102        """ 
     
    75109        if not chunk_size: 
    76110            chunk_size = UploadedFile.DEFAULT_CHUNK_SIZE 
    77         return self.file_size < chunk_size 
    78  
    79     # Abstract methods; subclasses *must* default read() and probably should 
     111        return self.size > chunk_size 
     112 
     113    # Abstract methods; subclasses *must* define read() and probably should 
    80114    # define open/close. 
    81115    def read(self, num_bytes=None): 
     
    88122        pass 
    89123 
     124    def xreadlines(self): 
     125        return self 
     126 
     127    def readlines(self): 
     128        return list(self.xreadlines()) 
     129 
     130    def __iter__(self): 
     131        # Iterate over this file-like object by newlines 
     132        buffer_ = None 
     133        for chunk in self.chunks(): 
     134            chunk_buffer = StringIO(chunk) 
     135 
     136            for line in chunk_buffer: 
     137                if buffer_: 
     138                    line = buffer_ + line 
     139                    buffer_ = None 
     140 
     141                # If this is the end of a line, yield 
     142                # otherwise, wait for the next round 
     143                if line[-1] in ('\n', '\r'): 
     144                    yield line 
     145                else: 
     146                    buffer_ = line 
     147 
     148        if buffer_ is not None: 
     149            yield buffer_ 
     150 
    90151    # Backwards-compatible support for uploaded-files-as-dictionaries. 
    91152    def __getitem__(self, key): 
    92         import warnings 
    93153        warnings.warn( 
    94154            message = "The dictionary access of uploaded file objects is deprecated. Use the new object interface instead.", 
     
    97157        ) 
    98158        backwards_translate = { 
    99             'filename': 'file_name', 
     159            'filename': 'name', 
    100160            'content-type': 'content_type', 
    101            
     161       
    102162 
    103163        if key == 'content': 
    104164            return self.read() 
    105165        elif key == 'filename': 
    106             return self.file_name 
     166            return self.name 
    107167        elif key == 'content-type': 
    108168            return self.content_type 
     
    114174    A file uploaded to a temporary location (i.e. stream-to-disk). 
    115175    """ 
    116  
    117     def __init__(self, file, file_name, content_type, file_size, charset): 
    118         super(TemporaryUploadedFile, self).__init__(file_name, content_type, file_size, charset) 
    119         self.file = file 
    120         self.path = file.name 
    121         self.file.seek(0
     176    def __init__(self, name, content_type, size, charset): 
     177        super(TemporaryUploadedFile, self).__init__(name, content_type, size, charset) 
     178        if settings.FILE_UPLOAD_TEMP_DIR: 
     179            self._file = tempfile.NamedTemporaryFile(suffix='.upload', dir=settings.FILE_UPLOAD_TEMP_DIR) 
     180        else: 
     181            self._file = tempfile.NamedTemporaryFile(suffix='.upload'
    122182 
    123183    def temporary_file_path(self): 
     
    125185        Returns the full path of this file. 
    126186        """ 
    127         return self.path 
    128  
    129     def read(self, *args, **kwargs): 
    130         return self.file.read(*args, **kwargs) 
    131  
    132     def open(self): 
    133         self.seek(0) 
    134  
    135     def seek(self, *args, **kwargs): 
    136         self.file.seek(*args, **kwargs) 
     187        return self.name 
     188     
     189    # Most methods on this object get proxied to NamedTemporaryFile. 
     190    # We can't directly subclass because NamedTemporaryFile is actually a 
     191    # factory function 
     192    def read(self, *args):          return self._file.read(*args) 
     193    def seek(self, offset):         return self._file.seek(offset) 
     194    def write(self, s):             return self._file.write(s) 
     195    def close(self):                return self._file.close() 
     196    def __iter__(self):             return iter(self._file) 
     197    def readlines(self, size=None): return self._file.readlines(size) 
     198    def xreadlines(self):           return self._file.xreadlines() 
    137199 
    138200class InMemoryUploadedFile(UploadedFile): 
     
    140202    A file uploaded into memory (i.e. stream-to-memory). 
    141203    """ 
    142     def __init__(self, file, field_name, file_name, content_type, file_size, charset): 
    143         super(InMemoryUploadedFile, self).__init__(file_name, content_type, file_size, charset) 
     204    def __init__(self, file, field_name, name, content_type, size, charset): 
     205        super(InMemoryUploadedFile, self).__init__(name, content_type, size, charset) 
    144206        self.file = file 
    145207        self.field_name = field_name 
     
    155217        return self.file.read(*args, **kwargs) 
    156218 
    157     def chunk(self, chunk_size=None): 
     219    def chunks(self, chunk_size=None): 
    158220        self.file.seek(0) 
    159221        yield self.read() 
     
    169231    def __init__(self, name, content, content_type='text/plain'): 
    170232        self.file = StringIO(content or '') 
    171         self.file_name = name 
     233        self.name = name 
    172234        self.field_name = None 
    173         self.file_size = len(content or '') 
     235        self.size = len(content or '') 
    174236        self.content_type = content_type 
    175237        self.charset = None 
  • django/trunk/django/core/files/uploadhandler.py

    r7817 r7859  
    133133        """ 
    134134        super(TemporaryFileUploadHandler, self).new_file(file_name, *args, **kwargs) 
    135         self.file = TemporaryFile(settings.FILE_UPLOAD_TEMP_DIR) 
    136         self.write = self.file.write 
     135        self.file = TemporaryUploadedFile(self.file_name, self.content_type, 0, self.charset) 
    137136 
    138137    def receive_data_chunk(self, raw_data, start): 
    139         self.write(raw_data) 
     138        self.file.write(raw_data) 
    140139 
    141140    def file_complete(self, file_size): 
    142141        self.file.seek(0) 
    143         return TemporaryUploadedFile( 
    144             file = self.file,  
    145             file_name = self.file_name,  
    146             content_type = self.content_type,  
    147             file_size = file_size,  
    148             charset = self.charset 
    149         ) 
     142        self.file.size = file_size 
     143        return self.file 
    150144 
    151145class MemoryFileUploadHandler(FileUploadHandler): 
     
    190184            file = self.file, 
    191185            field_name = self.field_name, 
    192             file_name = self.file_name, 
     186            name = self.file_name, 
    193187            content_type = self.content_type, 
    194             file_size = file_size, 
     188            size = file_size, 
    195189            charset = self.charset 
    196190        ) 
    197191 
    198 class TemporaryFile(object): 
    199     """ 
    200     A temporary file that tries to delete itself when garbage collected. 
    201     """ 
    202     def __init__(self, dir): 
    203         if not dir: 
    204             dir = tempfile.gettempdir() 
    205         try: 
    206             (fd, name) = tempfile.mkstemp(suffix='.upload', dir=dir) 
    207             self.file = os.fdopen(fd, 'w+b') 
    208         except (OSError, IOError): 
    209             raise OSError("Could not create temporary file for uploading, have you set settings.FILE_UPLOAD_TEMP_DIR correctly?") 
    210         self.name = name 
    211  
    212     def __getattr__(self, name): 
    213         a = getattr(self.__dict__['file'], name) 
    214         if type(a) != type(0): 
    215             setattr(self, name, a) 
    216         return a 
    217  
    218     def __del__(self): 
    219         try: 
    220             os.unlink(self.name) 
    221         except OSError: 
    222             pass 
    223192 
    224193def load_handler(path, *args, **kwargs): 
  • django/trunk/django/db/models/base.py

    r7846 r7859  
    537537            fp = open(full_filename, 'wb') 
    538538            locks.lock(fp, locks.LOCK_EX) 
    539             for chunk in raw_field.chunk(): 
     539            for chunk in raw_field.chunks(): 
    540540                fp.write(chunk) 
    541541            locks.unlock(fp) 
  • django/trunk/django/db/models/fields/__init__.py

    r7814 r7859  
    767767        "Returns field's value prepared for saving into a database." 
    768768        # Need to convert UploadedFile objects provided via a form to unicode for database insertion 
    769         if value is None: 
     769        if hasattr(value, 'name'): 
     770            return value.name 
     771        elif value is None: 
    770772            return None 
    771         return unicode(value) 
     773        else: 
     774            return unicode(value) 
    772775 
    773776    def get_manipulator_fields(self, opts, manipulator, change, name_prefix='', rel=False, follow=True): 
     
    843846            # do so for us. 
    844847            try: 
    845                 file_name = file.file_name 
     848                file_name = file.name 
    846849            except AttributeError: 
    847850                file_name = file['filename'] 
     
    858861 
    859862    def save_form_data(self, instance, data): 
    860         from django.newforms.fields import UploadedFile 
     863        from django.core.files.uploadedfile import UploadedFile 
    861864        if data and isinstance(data, UploadedFile): 
    862             getattr(instance, "save_%s_file" % self.name)(data.filename, data.data, save=False) 
     865            getattr(instance, "save_%s_file" % self.name)(data.name, data, save=False) 
    863866 
    864867    def formfield(self, **kwargs): 
  • django/trunk/django/newforms/fields.py

    r7814 r7859  
    2828from util import ErrorList, ValidationError 
    2929from widgets import TextInput, PasswordInput, HiddenInput, MultipleHiddenInput, FileInput, CheckboxInput, Select, NullBooleanSelect, SelectMultiple, DateTimeInput 
    30  
     30from django.core.files.uploadedfile import SimpleUploadedFile as UploadedFile 
    3131 
    3232__all__ = ( 
     
    420420    URL_VALIDATOR_USER_AGENT = 'Django (http://www.djangoproject.com/)' 
    421421 
    422 class UploadedFile(StrAndUnicode): 
    423     "A wrapper for files uploaded in a FileField" 
    424     def __init__(self, filename, data): 
    425         self.filename = filename 
    426         self.data = data 
    427  
    428     def __unicode__(self): 
    429         """ 
    430         The unicode representation is the filename, so that the pre-database-insertion 
    431         logic can use UploadedFile objects 
    432         """ 
    433         return self.filename 
    434422 
    435423class FileField(Field): 
     
    461449                stacklevel = 2 
    462450            ) 
     451            data = UploadedFile(data['filename'], data['content']) 
    463452 
    464453        try: 
    465             file_name = data.file_name 
    466             file_size = data.file_size 
     454            file_name = data.name 
     455            file_size = data.size 
    467456        except AttributeError: 
    468             try: 
    469                 file_name = data.get('filename') 
    470                 file_size = bool(data['content']) 
    471             except (AttributeError, KeyError): 
    472                 raise ValidationError(self.error_messages['invalid']) 
     457            raise ValidationError(self.error_messages['invalid']) 
    473458 
    474459        if not file_name: 
     
    477462            raise ValidationError(self.error_messages['empty']) 
    478463 
    479         return UploadedFile(file_name, data) 
     464        return data 
    480465 
    481466class ImageField(FileField): 
  • django/trunk/django/oldforms/__init__.py

    r7814 r7859  
    687687                raise validators.CriticalValidationError, upload_errors 
    688688        try: 
    689             file_size = new_data.file_size 
     689            file_size = new_data.size 
    690690        except AttributeError: 
    691691            file_size = len(new_data['content']) 
  • django/trunk/docs/newforms.txt

    r7845 r7859  
    13321332    * Error message keys: ``required``, ``invalid``, ``missing``, ``empty`` 
    13331333 
    1334 An ``UploadedFile`` object has two attributes: 
    1335  
    1336     ======================  ==================================================== 
    1337     Attribute               Description 
    1338     ======================  ==================================================== 
    1339     ``filename``            The name of the file, provided by the uploading 
    1340                             client. 
    1341                              
    1342     ``content``             The array of bytes comprising the file content. 
    1343     ======================  ==================================================== 
    1344  
    1345 The string representation of an ``UploadedFile`` is the same as the filename 
    1346 attribute. 
     1334To learn more about the ``UploadedFile`` object, see the `file uploads documentation`_. 
    13471335 
    13481336When you use a ``FileField`` in a form, you must also remember to 
    13491337`bind the file data to the form`_. 
    13501338 
     1339.. _file uploads documentation: ../upload_handling/ 
    13511340.. _`bind the file data to the form`: `Binding uploaded files to a form`_ 
    13521341 
  • django/trunk/docs/settings.txt

    r7844 r7859  
    280280The database backend to use. The build-in database backends are 
    281281``'postgresql_psycopg2'``, ``'postgresql'``, ``'mysql'``, ``'mysql_old'``, 
    282 ``'sqlite3'``, ``'oracle'``, and ``'oracle'``. 
     282``'sqlite3'``, and ``'oracle'``. 
    283283 
    284284In the Django development version, you can use a database backend that doesn't 
  • django/trunk/docs/upload_handling.txt

    r7827 r7859  
    2323        title = forms.CharField(max_length=50) 
    2424        file  = forms.FileField() 
    25          
     25 
    2626A view handling this form will receive the file data in ``request.FILES``, which 
    2727is a dictionary containing a key for each ``FileField`` (or ``ImageField``, or 
     
    6565        Read the entire uploaded data from the file. Be careful with this 
    6666        method: if the uploaded file is huge it can overwhelm your system if you 
    67         try to read it into memory. You'll probably want to use ``chunk()`` 
     67        try to read it into memory. You'll probably want to use ``chunks()`` 
    6868        instead; see below. 
    69          
     69 
    7070    ``UploadedFile.multiple_chunks()`` 
    7171        Returns ``True`` if the uploaded file is big enough to require 
    7272        reading in multiple chunks. By default this will be any file 
    7373        larger than 2.5 megabytes, but that's configurable; see below. 
    74      
     74 
    7575    ``UploadedFile.chunk()`` 
    7676        A generator returning chunks of the file. If ``multiple_chunks()`` is 
    7777        ``True``, you should use this method in a loop instead of ``read()``. 
    78          
     78 
    7979        In practice, it's often easiest simply to use ``chunks()`` all the time; 
    8080        see the example below. 
    81      
     81 
    8282    ``UploadedFile.file_name`` 
    8383        The name of the uploaded file (e.g. ``my_file.txt``). 
    84          
     84 
    8585    ``UploadedFile.file_size`` 
    8686        The size, in bytes, of the uploaded file. 
    87          
     87 
    8888There are a few other methods and attributes available on ``UploadedFile`` 
    8989objects; see `UploadedFile objects`_ for a complete reference. 
    9090 
    9191Putting it all together, here's a common way you might handle an uploaded file:: 
    92      
     92 
    9393    def handle_uploaded_file(f): 
    9494        destination = open('some/file/name.txt', 'wb') 
     
    127127        into memory. Files larger than ``FILE_UPLOAD_MAX_MEMORY_SIZE`` 
    128128        will be streamed to disk. 
    129          
     129 
    130130        Defaults to 2.5 megabytes. 
    131          
     131 
    132132    ``FILE_UPLOAD_TEMP_DIR`` 
    133133        The directory where uploaded files larger than ``FILE_UPLOAD_TEMP_DIR`` 
    134134        will be stored. 
    135          
     135 
    136136        Defaults to your system's standard temporary directory (i.e. ``/tmp`` on 
    137137        most Unix-like systems). 
    138          
     138 
    139139    ``FILE_UPLOAD_HANDLERS`` 
    140140        The actual handlers for uploaded files. Changing this setting 
     
    142142        Django's upload process. See `upload handlers`_, below, 
    143143        for details. 
    144          
     144 
    145145        Defaults to:: 
    146          
     146 
    147147            ("django.core.files.uploadhandler.MemoryFileUploadHandler", 
    148148             "django.core.files.uploadhandler.TemporaryFileUploadHandler",) 
    149              
     149 
    150150        Which means "try to upload to memory first, then fall back to temporary 
    151151        files." 
     
    162162        ``num_bytes`` is ``None``. 
    163163 
    164     ``UploadedFile.chunk(self, chunk_size=None)`` 
     164    ``UploadedFile.chunks(self, chunk_size=None)`` 
    165165        A generator yielding small chunks from the file. If ``chunk_size`` isn't 
    166         given, chunks will be 64 kb
     166        given, chunks will be 64 KB
    167167 
    168168    ``UploadedFile.multiple_chunks(self, chunk_size=None)`` 
    169169        Returns ``True`` if you can expect more than one chunk when calling 
    170         ``UploadedFile.chunk(self, chunk_size)``. 
     170        ``UploadedFile.chunks(self, chunk_size)``. 
    171171 
    172172    ``UploadedFile.file_size`` 
    173173        The size, in bytes, of the uploaded file. 
    174      
     174 
    175175    ``UploadedFile.file_name`` 
    176176        The name of the uploaded file as provided by the user. 
    177      
     177 
    178178    ``UploadedFile.content_type`` 
    179179        The content-type header uploaded with the file (e.g. ``text/plain`` or 
     
    182182        validate that the file contains the content that the content-type header 
    183183        claims -- "trust but verify." 
    184      
     184 
    185185    ``UploadedFile.charset`` 
    186186        For ``text/*`` content-types, the character set (i.e. ``utf8``) supplied 
    187187        by the browser. Again, "trust but verify" is the best policy here. 
    188188 
     189    ``UploadedFile.__iter__()`` 
     190        Iterates over the lines in the file. 
     191 
    189192    ``UploadedFile.temporary_file_path()`` 
    190193        Only files uploaded onto disk will have this method; it returns the full 
    191194        path to the temporary uploaded file. 
     195 
    192196 
    193197Upload Handlers 
  • django/trunk/tests/modeltests/model_forms/models.py

    r7814 r7859  
    804804True 
    805805>>> type(f.cleaned_data['file']) 
    806 <class 'django.newforms.fields.UploadedFile'> 
     806<class 'django.core.files.uploadedfile.SimpleUploadedFile'> 
    807807>>> instance = f.save() 
    808808>>> instance.file 
     
    815815True 
    816816>>> type(f.cleaned_data['file']) 
    817 <class 'django.newforms.fields.UploadedFile'> 
     817<class 'django.core.files.uploadedfile.SimpleUploadedFile'> 
    818818>>> instance = f.save() 
    819819>>> instance.file 
     
    907907True 
    908908>>> type(f.cleaned_data['image']) 
    909 <class 'django.newforms.fields.UploadedFile'> 
     909<class 'django.core.files.uploadedfile.SimpleUploadedFile'> 
    910910>>> instance = f.save() 
    911911>>> instance.image 
     
    919919True 
    920920>>> type(f.cleaned_data['image']) 
    921 <class 'django.newforms.fields.UploadedFile'> 
     921<class 'django.core.files.uploadedfile.SimpleUploadedFile'> 
    922922>>> instance = f.save() 
    923923>>> instance.image 
  • django/trunk/tests/regressiontests/file_uploads/views.py

    r7858 r7859  
    1616        # If a file is posted, the dummy client should only post the file name, 
    1717        # not the full path. 
    18         if os.path.dirname(form_data['file_field'].file_name) != '': 
     18        if os.path.dirname(form_data['file_field'].name) != '': 
    1919            return HttpResponseServerError()             
    2020        return HttpResponse('') 
     
    3030 
    3131    # Check to see if unicode names worked out. 
    32     if not request.FILES['file_unicode'].file_name.endswith(u'test_\u4e2d\u6587_Orl\xe9ans.jpg'): 
     32    if not request.FILES['file_unicode'].name.endswith(u'test_\u4e2d\u6587_Orl\xe9ans.jpg'): 
    3333        return HttpResponseServerError() 
    3434 
     
    5252    Simple view to echo back info about uploaded files for tests. 
    5353    """ 
    54     r = dict([(k, f.file_name) for k, f in request.FILES.items()]) 
     54    r = dict([(k, f.name) for k, f in request.FILES.items()]) 
    5555    return HttpResponse(simplejson.dumps(r)) 
    5656     
  • django/trunk/tests/regressiontests/forms/fields.py

    r7814 r7859  
    801801 
    802802>>> type(f.clean(SimpleUploadedFile('name', 'Some File Content'))) 
    803 <class 'django.newforms.fields.UploadedFile'> 
     803<class 'django.core.files.uploadedfile.SimpleUploadedFile'> 
    804804 
    805805>>> type(f.clean(SimpleUploadedFile('name', 'Some File Content'), 'files/test4.pdf')) 
    806 <class 'django.newforms.fields.UploadedFile'> 
     806<class 'django.core.files.uploadedfile.SimpleUploadedFile'> 
    807807 
    808808# URLField ##################################################################