Index: django/db/models/base.py =================================================================== --- django/db/models/base.py (revision 7857) +++ django/db/models/base.py (working copy) @@ -536,7 +536,7 @@ # This is a normal uploadedfile that we can stream. fp = open(full_filename, 'wb') locks.lock(fp, locks.LOCK_EX) - for chunk in raw_field.chunk(): + for chunk in raw_field.chunks(): fp.write(chunk) locks.unlock(fp) fp.close() Index: django/db/models/fields/__init__.py =================================================================== --- django/db/models/fields/__init__.py (revision 7857) +++ django/db/models/fields/__init__.py (working copy) @@ -766,9 +766,12 @@ def get_db_prep_save(self, value): "Returns field's value prepared for saving into a database." # Need to convert UploadedFile objects provided via a form to unicode for database insertion - if value is None: + if hasattr(value, 'name'): + return value.name + elif value is None: return None - return unicode(value) + else: + return unicode(value) def get_manipulator_fields(self, opts, manipulator, change, name_prefix='', rel=False, follow=True): field_list = Field.get_manipulator_fields(self, opts, manipulator, change, name_prefix, rel, follow) @@ -842,7 +845,7 @@ # We don't need to raise a warning because Model._save_FIELD_file will # do so for us. try: - file_name = file.file_name + file_name = file.name except AttributeError: file_name = file['filename'] @@ -857,9 +860,9 @@ return os.path.normpath(f) def save_form_data(self, instance, data): - from django.newforms.fields import UploadedFile + from django.core.files.uploadedfile import UploadedFile if data and isinstance(data, UploadedFile): - getattr(instance, "save_%s_file" % self.name)(data.filename, data.data, save=False) + getattr(instance, "save_%s_file" % self.name)(data.name, data, save=False) def formfield(self, **kwargs): defaults = {'form_class': forms.FileField} Index: django/oldforms/__init__.py =================================================================== --- django/oldforms/__init__.py (revision 7857) +++ django/oldforms/__init__.py (working copy) @@ -686,7 +686,7 @@ if upload_errors: raise validators.CriticalValidationError, upload_errors try: - file_size = new_data.file_size + file_size = new_data.size except AttributeError: file_size = len(new_data['content']) if not file_size: Index: django/core/files/uploadedfile.py =================================================================== --- django/core/files/uploadedfile.py (revision 7857) +++ django/core/files/uploadedfile.py (working copy) @@ -3,13 +3,18 @@ """ import os +import tempfile +import warnings try: from cStringIO import StringIO except ImportError: from StringIO import StringIO -__all__ = ('UploadedFile', 'TemporaryUploadedFile', 'InMemoryUploadedFile') +from django.conf import settings + +__all__ = ('UploadedFile', 'TemporaryUploadedFile', 'InMemoryUploadedFile', 'SimpleUploadedFile') + class UploadedFile(object): """ A abstract uploaded file (``TemporaryUploadedFile`` and @@ -20,34 +25,87 @@ """ DEFAULT_CHUNK_SIZE = 64 * 2**10 - def __init__(self, file_name=None, content_type=None, file_size=None, charset=None): - self.file_name = file_name - self.file_size = file_size + def __init__(self, name=None, content_type=None, size=None, charset=None): + self.name = name + self.size = size self.content_type = content_type self.charset = charset def __repr__(self): - return "<%s: %s (%s)>" % (self.__class__.__name__, self.file_name, self.content_type) + return "<%s: %s (%s)>" % (self.__class__.__name__, self.name, self.content_type) - def _set_file_name(self, name): + def _get_name(self): + return self._name + + def _set_name(self, name): # Sanitize the file name so that it can't be dangerous. if name is not None: # Just use the basename of the file -- anything else is dangerous. name = os.path.basename(name) - + # File names longer than 255 characters can cause problems on older OSes. if len(name) > 255: name, ext = os.path.splitext(name) name = name[:255 - len(ext)] + ext - - self._file_name = name - + + self._name = name + + name = property(_get_name, _set_name) + + def _get_file_name(self): - return self._file_name - - file_name = property(_get_file_name, _set_file_name) + warnings.warn( + message = "To access the file name, please use the .name attribute.", + category = DeprecationWarning, + stacklevel = 2 + ) + return self.name - def chunk(self, chunk_size=None): + def _set_file_name(self, name): + warnings.warn( + message = "To access the file name, please use the .name attribute.", + category = DeprecationWarning, + stacklevel = 2 + ) + self.name = name + + file_name = filename = property(_get_file_name, _set_file_name) + + + def _get_file_size(self): + warnings.warn( + message = "To access the file size, please use the .size attribute.", + category = DeprecationWarning, + stacklevel = 2 + ) + return self.size + + def _set_file_size(self, size): + warnings.warn( + message = "To access the file size, please use the .size attribute.", + category = DeprecationWarning, + stacklevel = 2 + ) + self.size = size + + file_size = property(_get_file_size, _set_file_size) + + + def _get_data(self): + warnings.warn( + message = "To access the data, please use the new UploadedFile API.", + category = DeprecationWarning, + stacklevel = 2 + ) + data = self.read() + if hasattr(self, 'seek'): + self.seek(0) + return data + + data = property(_get_data) + + + def chunks(self, chunk_size=None): """ Read the file and yield chucks of ``chunk_size`` bytes (defaults to ``UploadedFile.DEFAULT_CHUNK_SIZE``). @@ -58,12 +116,21 @@ if hasattr(self, 'seek'): self.seek(0) # Assume the pointer is at zero... - counter = self.file_size + counter = self.size while counter > 0: yield self.read(chunk_size) counter -= chunk_size + def chunk(self, *args, **kwargs): + warnings.warn( + message = "Please use .chunks() instead of .chunk().", + category = DeprecationWarning, + stacklevel = 2 + ) + for chunk in self.chunks(): + yield chunk + def multiple_chunks(self, chunk_size=None): """ Returns ``True`` if you can expect multiple chunks. @@ -74,9 +141,9 @@ """ if not chunk_size: chunk_size = UploadedFile.DEFAULT_CHUNK_SIZE - return self.file_size < chunk_size + return self.size > chunk_size - # Abstract methods; subclasses *must* default read() and probably should + # Abstract methods; subclasses *must* define read() and probably should # define open/close. def read(self, num_bytes=None): raise NotImplementedError() @@ -87,60 +154,96 @@ def close(self): pass + def xreadlines(self): + return self + + def readlines(self): + return list(self.xreadlines()) + + def __iter__(self): + # Iterate over this file-like object by newlines + buffer_ = None + for chunk in self.chunks(): + chunk_buffer = StringIO(chunk) + + for line in chunk_buffer: + if buffer_: + line = buffer_ + line + buffer_ = None + + # If this is the end of a line, yield + # otherwise, wait for the next round + if line[-1] in ('\n', '\r'): + yield line + else: + buffer_ = line + + if buffer_ is not None: + yield buffer_ + # Backwards-compatible support for uploaded-files-as-dictionaries. def __getitem__(self, key): - import warnings warnings.warn( message = "The dictionary access of uploaded file objects is deprecated. Use the new object interface instead.", category = DeprecationWarning, stacklevel = 2 ) backwards_translate = { - 'filename': 'file_name', + 'filename': 'name', 'content-type': 'content_type', } if key == 'content': return self.read() elif key == 'filename': - return self.file_name + return self.name elif key == 'content-type': return self.content_type else: return getattr(self, key) + +def _proxy_to_file(method_name): + # Function factory to build proxy methods for the _file object. + def __inner_function(self, *args, **kwargs): + return getattr(self._file, method_name)(*args, **kwargs) + __inner_function.__name__ = method_name + return __inner_function + + class TemporaryUploadedFile(UploadedFile): """ A file uploaded to a temporary location (i.e. stream-to-disk). """ + def __init__(self, name, content_type, size, charset): + super(TemporaryUploadedFile, self).__init__(name, content_type, size, charset) + if settings.FILE_UPLOAD_TEMP_DIR: + self._file = tempfile.NamedTemporaryFile(suffix='.upload', dir=settings.FILE_UPLOAD_TEMP_DIR) + else: + self._file = tempfile.NamedTemporaryFile(suffix='.upload') - def __init__(self, file, file_name, content_type, file_size, charset): - super(TemporaryUploadedFile, self).__init__(file_name, content_type, file_size, charset) - self.file = file - self.path = file.name - self.file.seek(0) - def temporary_file_path(self): """ Returns the full path of this file. """ - return self.path + return self.name - def read(self, *args, **kwargs): - return self.file.read(*args, **kwargs) + # Functions from the local file object + read = _proxy_to_file('read') + seek = _proxy_to_file('seek') + write = _proxy_to_file('write') + close = _proxy_to_file('close') + __iter__ = _proxy_to_file('__iter__') + readlines = _proxy_to_file('readlines') + xreadlines = _proxy_to_file('xreadlines') - def open(self): - self.seek(0) - def seek(self, *args, **kwargs): - self.file.seek(*args, **kwargs) - class InMemoryUploadedFile(UploadedFile): """ A file uploaded into memory (i.e. stream-to-memory). """ - def __init__(self, file, field_name, file_name, content_type, file_size, charset): - super(InMemoryUploadedFile, self).__init__(file_name, content_type, file_size, charset) + def __init__(self, file, field_name, name, content_type, size, charset): + super(InMemoryUploadedFile, self).__init__(name, content_type, size, charset) self.file = file self.field_name = field_name self.file.seek(0) @@ -154,7 +257,7 @@ def read(self, *args, **kwargs): return self.file.read(*args, **kwargs) - def chunk(self, chunk_size=None): + def chunks(self, chunk_size=None): self.file.seek(0) yield self.read() @@ -168,9 +271,9 @@ """ def __init__(self, name, content, content_type='text/plain'): self.file = StringIO(content or '') - self.file_name = name + self.name = name self.field_name = None - self.file_size = len(content or '') + self.size = len(content or '') self.content_type = content_type self.charset = None self.file.seek(0) Index: django/core/files/uploadhandler.py =================================================================== --- django/core/files/uploadhandler.py (revision 7857) +++ django/core/files/uploadhandler.py (working copy) @@ -132,21 +132,15 @@ Create the file object to append to as data is coming in. """ super(TemporaryFileUploadHandler, self).new_file(file_name, *args, **kwargs) - self.file = TemporaryFile(settings.FILE_UPLOAD_TEMP_DIR) - self.write = self.file.write + self.file = TemporaryUploadedFile(self.file_name, self.content_type, 0, self.charset) def receive_data_chunk(self, raw_data, start): - self.write(raw_data) + self.file.write(raw_data) def file_complete(self, file_size): self.file.seek(0) - return TemporaryUploadedFile( - file = self.file, - file_name = self.file_name, - content_type = self.content_type, - file_size = file_size, - charset = self.charset - ) + self.file.size = file_size + return self.file class MemoryFileUploadHandler(FileUploadHandler): """ @@ -189,38 +183,13 @@ return InMemoryUploadedFile( file = self.file, field_name = self.field_name, - file_name = self.file_name, + name = self.file_name, content_type = self.content_type, - file_size = file_size, + size = file_size, charset = self.charset ) -class TemporaryFile(object): - """ - A temporary file that tries to delete itself when garbage collected. - """ - def __init__(self, dir): - if not dir: - dir = tempfile.gettempdir() - try: - (fd, name) = tempfile.mkstemp(suffix='.upload', dir=dir) - self.file = os.fdopen(fd, 'w+b') - except (OSError, IOError): - raise OSError("Could not create temporary file for uploading, have you set settings.FILE_UPLOAD_TEMP_DIR correctly?") - self.name = name - def __getattr__(self, name): - a = getattr(self.__dict__['file'], name) - if type(a) != type(0): - setattr(self, name, a) - return a - - def __del__(self): - try: - os.unlink(self.name) - except OSError: - pass - def load_handler(path, *args, **kwargs): """ Given a path to a handler, return an instance of that handler. Index: django/newforms/fields.py =================================================================== --- django/newforms/fields.py (revision 7857) +++ django/newforms/fields.py (working copy) @@ -27,8 +27,8 @@ from util import ErrorList, ValidationError from widgets import TextInput, PasswordInput, HiddenInput, MultipleHiddenInput, FileInput, CheckboxInput, Select, NullBooleanSelect, SelectMultiple, DateTimeInput +from django.core.files.uploadedfile import SimpleUploadedFile as UploadedFile - __all__ = ( 'Field', 'CharField', 'IntegerField', 'DEFAULT_DATE_INPUT_FORMATS', 'DateField', @@ -419,19 +419,7 @@ # It's OK if Django settings aren't configured. URL_VALIDATOR_USER_AGENT = 'Django (http://www.djangoproject.com/)' -class UploadedFile(StrAndUnicode): - "A wrapper for files uploaded in a FileField" - def __init__(self, filename, data): - self.filename = filename - self.data = data - def __unicode__(self): - """ - The unicode representation is the filename, so that the pre-database-insertion - logic can use UploadedFile objects - """ - return self.filename - class FileField(Field): widget = FileInput default_error_messages = { @@ -460,23 +448,20 @@ category = DeprecationWarning, stacklevel = 2 ) + data = UploadedFile(data['filename'], data['content']) try: - file_name = data.file_name - file_size = data.file_size + file_name = data.name + file_size = data.size except AttributeError: - try: - file_name = data.get('filename') - file_size = bool(data['content']) - except (AttributeError, KeyError): - raise ValidationError(self.error_messages['invalid']) + raise ValidationError(self.error_messages['invalid']) if not file_name: raise ValidationError(self.error_messages['invalid']) if not file_size: raise ValidationError(self.error_messages['empty']) - return UploadedFile(file_name, data) + return data class ImageField(FileField): default_error_messages = { Index: tests/modeltests/model_forms/models.py =================================================================== --- tests/modeltests/model_forms/models.py (revision 7857) +++ tests/modeltests/model_forms/models.py (working copy) @@ -803,7 +803,7 @@ >>> f.is_valid() True >>> type(f.cleaned_data['file']) - + >>> instance = f.save() >>> instance.file u'...test1.txt' @@ -814,7 +814,7 @@ >>> f.is_valid() True >>> type(f.cleaned_data['file']) - + >>> instance = f.save() >>> instance.file u'...test1.txt' @@ -906,7 +906,7 @@ >>> f.is_valid() True >>> type(f.cleaned_data['image']) - + >>> instance = f.save() >>> instance.image u'...test.png' @@ -918,7 +918,7 @@ >>> f.is_valid() True >>> type(f.cleaned_data['image']) - + >>> instance = f.save() >>> instance.image u'...test.png' Index: tests/regressiontests/forms/fields.py =================================================================== --- tests/regressiontests/forms/fields.py (revision 7857) +++ tests/regressiontests/forms/fields.py (working copy) @@ -800,10 +800,10 @@ ValidationError: [u'The submitted file is empty.'] >>> type(f.clean(SimpleUploadedFile('name', 'Some File Content'))) - + >>> type(f.clean(SimpleUploadedFile('name', 'Some File Content'), 'files/test4.pdf')) - + # URLField ################################################################## Index: docs/upload_handling.txt =================================================================== --- docs/upload_handling.txt (revision 7857) +++ docs/upload_handling.txt (working copy) @@ -22,7 +22,7 @@ class UploadFileForm(forms.Form): title = forms.CharField(max_length=50) file = forms.FileField() - + A view handling this form will receive the file data in ``request.FILES``, which is a dictionary containing a key for each ``FileField`` (or ``ImageField``, or other ``FileField`` subclass) in the form. So the data from the above form would @@ -64,32 +64,32 @@ ``UploadedFile.read()`` Read the entire uploaded data from the file. Be careful with this method: if the uploaded file is huge it can overwhelm your system if you - try to read it into memory. You'll probably want to use ``chunk()`` + try to read it into memory. You'll probably want to use ``chunks()`` instead; see below. - + ``UploadedFile.multiple_chunks()`` Returns ``True`` if the uploaded file is big enough to require reading in multiple chunks. By default this will be any file larger than 2.5 megabytes, but that's configurable; see below. - + ``UploadedFile.chunk()`` A generator returning chunks of the file. If ``multiple_chunks()`` is ``True``, you should use this method in a loop instead of ``read()``. - + In practice, it's often easiest simply to use ``chunks()`` all the time; see the example below. - + ``UploadedFile.file_name`` The name of the uploaded file (e.g. ``my_file.txt``). - + ``UploadedFile.file_size`` The size, in bytes, of the uploaded file. - + There are a few other methods and attributes available on ``UploadedFile`` objects; see `UploadedFile objects`_ for a complete reference. Putting it all together, here's a common way you might handle an uploaded file:: - + def handle_uploaded_file(f): destination = open('some/file/name.txt', 'wb') for chunk in f.chunks(): @@ -126,27 +126,27 @@ The maximum size, in bytes, for files that will be uploaded into memory. Files larger than ``FILE_UPLOAD_MAX_MEMORY_SIZE`` will be streamed to disk. - + Defaults to 2.5 megabytes. - + ``FILE_UPLOAD_TEMP_DIR`` The directory where uploaded files larger than ``FILE_UPLOAD_TEMP_DIR`` will be stored. - + Defaults to your system's standard temporary directory (i.e. ``/tmp`` on most Unix-like systems). - + ``FILE_UPLOAD_HANDLERS`` The actual handlers for uploaded files. Changing this setting allows complete customization -- even replacement -- of Django's upload process. See `upload handlers`_, below, for details. - + Defaults to:: - + ("django.core.files.uploadhandler.MemoryFileUploadHandler", "django.core.files.uploadhandler.TemporaryFileUploadHandler",) - + Which means "try to upload to memory first, then fall back to temporary files." @@ -161,35 +161,39 @@ Returns a byte string of length ``num_bytes``, or the complete file if ``num_bytes`` is ``None``. - ``UploadedFile.chunk(self, chunk_size=None)`` + ``UploadedFile.chunks(self, chunk_size=None)`` A generator yielding small chunks from the file. If ``chunk_size`` isn't - given, chunks will be 64 kb. + given, chunks will be 64 KB. ``UploadedFile.multiple_chunks(self, chunk_size=None)`` Returns ``True`` if you can expect more than one chunk when calling - ``UploadedFile.chunk(self, chunk_size)``. + ``UploadedFile.chunks(self, chunk_size)``. ``UploadedFile.file_size`` The size, in bytes, of the uploaded file. - + ``UploadedFile.file_name`` The name of the uploaded file as provided by the user. - + ``UploadedFile.content_type`` The content-type header uploaded with the file (e.g. ``text/plain`` or ``application/pdf``). Like any data supplied by the user, you shouldn't trust that the uploaded file is actually this type. You'll still need to validate that the file contains the content that the content-type header claims -- "trust but verify." - + ``UploadedFile.charset`` For ``text/*`` content-types, the character set (i.e. ``utf8``) supplied by the browser. Again, "trust but verify" is the best policy here. + ``UploadedFile.__iter__()`` + Iterates over the lines in the file. + ``UploadedFile.temporary_file_path()`` Only files uploaded onto disk will have this method; it returns the full path to the temporary uploaded file. + Upload Handlers =============== Index: docs/newforms.txt =================================================================== --- docs/newforms.txt (revision 7857) +++ docs/newforms.txt (working copy) @@ -1331,23 +1331,12 @@ * Validates that non-empty file data has been bound to the form. * Error message keys: ``required``, ``invalid``, ``missing``, ``empty`` -An ``UploadedFile`` object has two attributes: +To learn more about the ``UploadedFile`` object, see the `file uploads documentation`_. - ====================== ==================================================== - Attribute Description - ====================== ==================================================== - ``filename`` The name of the file, provided by the uploading - client. - - ``content`` The array of bytes comprising the file content. - ====================== ==================================================== - -The string representation of an ``UploadedFile`` is the same as the filename -attribute. - When you use a ``FileField`` in a form, you must also remember to `bind the file data to the form`_. +.. _file uploads documentation: ../upload_handling/ .. _`bind the file data to the form`: `Binding uploaded files to a form`_ ``FilePathField`` Index: docs/settings.txt =================================================================== --- docs/settings.txt (revision 7857) +++ docs/settings.txt (working copy) @@ -279,7 +279,7 @@ The database backend to use. The build-in database backends are ``'postgresql_psycopg2'``, ``'postgresql'``, ``'mysql'``, ``'mysql_old'``, -``'sqlite3'``, ``'oracle'``, and ``'oracle'``. +``'sqlite3'``, and ``'oracle'``. In the Django development version, you can use a database backend that doesn't ship with Django by setting ``DATABASE_ENGINE`` to a fully-qualified path (i.e.