Index: django/test/client.py =================================================================== --- django/test/client.py (revision 7354) +++ django/test/client.py (working copy) @@ -18,6 +18,25 @@ BOUNDARY = 'BoUnDaRyStRiNg' MULTIPART_CONTENT = 'multipart/form-data; boundary=%s' % BOUNDARY +class FakePayload(object): + """ + A wrapper around StringIO that restricts what can be read, + since data from the network can't be seeked and cannot + be read outside of its content length (or else we hang). + """ + def __init__(self, content): + self.__content = StringIO(content) + self.__len = len(content) + + def read(self, num_bytes=None): + if num_bytes is None: + num_bytes = self.__len or 1 + assert self.__len >= num_bytes, "Cannot read more than the available bytes from the HTTP incoming data." + content = self.__content.read(num_bytes) + self.__len -= num_bytes + return content + + class ClientHandler(BaseHandler): """ A HTTP Handler that can be used for testing purposes. @@ -230,7 +249,7 @@ 'CONTENT_TYPE': content_type, 'PATH_INFO': urllib.unquote(path), 'REQUEST_METHOD': 'POST', - 'wsgi.input': StringIO(post_data), + 'wsgi.input': FakePayload(post_data), } r.update(extra) Index: django/http/multipartparser.py =================================================================== --- django/http/multipartparser.py (revision 0) +++ django/http/multipartparser.py (revision 0) @@ -0,0 +1,585 @@ +""" +MultiPart parsing for file uploads. + +This object will take the file upload headers +and the file upload handler and chunk the upload +data for the handler to deal with. +""" +from django.utils.datastructures import MultiValueDict +from django.utils.encoding import force_unicode + +__all__ = ('MultiPartParser','MultiPartParserError','InputStreamExhausted') + +class MultiPartParserError(Exception): + pass + +class InputStreamExhausted(Exception): + """ No more reads are allowed from this device. """ + pass + +class MultiPartParser(object): + """ + A rfc2388 multipart/form-data parser. + + parse() reads the input stream in chunk_size chunks and returns a + tuple of (POST MultiValueDict, FILES MultiValueDict). If + file_upload_dir is defined files will be streamed to temporary + files in the specified directory. + """ + def __init__(self, META, input_data, upload_handlers, encoding=None): + """ + Initialize the MultiPartParser object. + + *META* -- The standard META dictionary in Django request objects. + *input_data* -- The raw post data, as a bytestring. + *upload_handler* -- An object of type UploadHandler + that performs operations on the uploaded + data. + *encoding* -- The encoding with which to treat the incoming data. + """ + # Import cgi utilities for (near) future use. + global parse_header, valid_boundary, settings + from django.conf import settings + from cgi import valid_boundary, parse_header + + ####### + # Check basic headers + ####### + + # + # Content-Type should containt multipart and the boundary information. + #### + + content_type = META.get('HTTP_CONTENT_TYPE', META.get('CONTENT_TYPE', '')) + if not content_type.startswith('multipart/'): + raise MultiPartParserError('Invalid Content-Type: %s' % + content_type) + + # Parse the header to get the boundary to split the parts. + ctypes, opts = parse_header(content_type) + boundary = opts.get('boundary') + if not boundary or not valid_boundary(boundary): + raise MultiPartParserError('Invalid boundary in multipart: %s' % + boundary) + + + # + # Content-Length should contain the length of the body we are about + # to receive. + #### + try: + content_length = int(META.get('HTTP_CONTENT_LENGTH', + META.get('CONTENT_LENGTH',0))) + except (ValueError, TypeError): + # For now set it to 0...we'll try again later on down. + content_length = 0 + + # If we have better knowledge of how much + # data is remaining in the request stream, + # we should use that. (modpython for instance) + #try: + # remaining = input_data.remaining + # if remaining is not None and \ + # (content_length is None or remaining < content_length): + # content_length = remaining + #except AttributeError: + # pass + + if not content_length: + # This means we shouldn't continue...raise an error. + raise MultiPartParserError("Invalid content length: %r" % content_length) + + self._boundary = boundary + self._input_data = input_data + + # For compatibility with low-level network APIs (with 32-bit integers), + # the chunk size should be < 2^31: + self._chunk_size = min(2147483647, *[x.chunk_size for x in upload_handlers + if x.chunk_size]) + + self._meta = META + self._encoding = encoding or settings.DEFAULT_CHARSET + self._content_length = content_length + self._upload_handlers = upload_handlers + + def parse(self): + """ + Parse the POST data and break it into a FILES MultiValueDict + and a POST MultiValueDict. + + *returns* -- A tuple containing the POST and FILES dictionary, + respectively. + """ + from django.core.files.fileuploadhandler import StopUpload, SkipFile + from django.http import QueryDict + + encoding = self._encoding + handlers = self._upload_handlers + + limited_input_data = LimitBytes(self._input_data, self._content_length) + + # See if the handler will want to take care of the parsing. + # This allows overriding everything if somebody wants it. + for handler in handlers: + result = handler.handle_raw_input(limited_input_data, + self._meta, + self._content_length, + self._boundary, + encoding) + if result is not None: + return result[0], result[1] + + # Create the data structures to be used later. + self._post = QueryDict('', mutable=True) + self._files = MultiValueDict() + + # Instantiate the parser and stream: + stream = LazyStream(ChunkIter(limited_input_data, self._chunk_size)) + for item_type, meta_data, stream in Parser(stream, self._boundary): + try: + disposition = meta_data['content-disposition'][1] + field_name = disposition['name'].strip() + except (KeyError, IndexError, AttributeError): + continue + + transfer_encoding = meta_data.get('content-transfer-encoding') + + field_name = force_unicode(field_name, encoding, errors='replace') + + if item_type == 'FIELD': + # This is a post field, we can just set it in the post + if transfer_encoding == 'base64': + raw_data = stream.read() + try: + data = str(raw_data).decode('base64') + except: + data = raw_data + else: + data = stream.read() + + self._post.appendlist(field_name, + force_unicode(data, encoding, errors='replace')) + elif item_type == 'FILE': + # This is a file, use the handler... + file_successful = True + file_name = self.IE_sanitize(disposition.get('filename')) + if not file_name: + continue + + file_name = force_unicode(file_name, encoding, errors='replace') + + content_type = meta_data.get('content-type', ('',))[0].strip() + try: + charset = meta_data.get('content-type', (0,{}))[1].get('charset', None) + except: + charset = None + + try: + content_length = int(meta_data.get('content-length')[0]) + except (IndexError, TypeError, ValueError): + content_length = None + + counter = 0 + try: + for handler in handlers: + retval = handler.new_file(field_name, file_name, + content_type, content_length, + charset) + if retval: + break + + for chunk in stream: + if transfer_encoding == 'base64': + # We only special-case base64 transfer encoding + try: + chunk = str(chunk).decode('base64') + except Exception, e: + # Since this is only a chunk, any error is an unfixable error. + raise MultiValueParseError("Could not decode base64 data: %r" % e) + + chunk_length = len(chunk) + counter += chunk_length + for handler in handlers: + chunk = handler.receive_data_chunk(chunk, + counter - chunk_length, + counter) + if chunk is None: + break + + except (StopUpload, SkipFile), e: + file_successful = False + if isinstance(e, SkipFile): + # Just use up the rest of this file... + stream.exhaust() + elif isinstance(e, StopUpload): + # Abort the parsing and break + parser.abort() + break + else: + # Only do this if the handler didn't raise an abort error + for handler in handlers: + file_obj = handler.file_complete(counter) + if file_obj: + # If it returns a file object, then set the files dict. + self._files.appendlist(force_unicode(field_name, + encoding, + errors='replace'), + file_obj) + break + else: + # If this is neither a FIELD or a FILE, just exhaust the stream. + stream.exhuast() + + # Make sure that the request data is all fed + limited_input_data.exhaust() + + # Signal that the upload has completed. + for handler in handlers: + retval = handler.upload_complete() + if retval: + break + + return self._post, self._files + + def IE_sanitize(self, filename): + """cleanup filename from IE full paths""" + return filename and filename[filename.rfind("\\")+1:].strip() + + +class LazyStream(object): + def __init__(self, producer, length=None): + """ + Every LazyStream must have a producer when instantiated. + + A producer is an iterable that returns a string each time it + is called. + """ + self._producer = producer + self._empty = False + self._leftover = '' + self.length = length + self.position = 0 + self._remaining = length + + def tell(self): + return self.position + + def read(self, size=None): + def parts(): + remaining = (size is not None and [size] or [self._remaining])[0] + # do the whole thing in one shot if no limit was provided. + if remaining is None: + yield ''.join(self) + return + + # otherwise do some bookkeeping to return exactly enough + # of the stream and stashing any extra content we get from + # the producer + while remaining != 0: + assert remaining > 0, 'remaining bytes to read should never go negative' + + chunk = self.next() + + emitting = chunk[:remaining] + self.unget(chunk[remaining:]) + remaining -= len(emitting) + yield emitting + + out = ''.join(parts()) + self.position += len(out) + return out + + def next(self): + """ + Used when the exact number of bytes to read is unimportant. + + This procedure just returns whatever is chunk is conveniently + returned from the iterator instead. Useful to avoid + unnecessary bookkeeping if performance is an issue. + """ + if self._leftover: + output = self._leftover + self.position += len(output) + self._leftover = '' + return output + else: + output = self._producer.next() + self.position += len(output) + return output + + def close(self): + """ + Used to invalidate/disable this lazy stream. + + Replaces the producer with an empty list. Any leftover bytes + that have already been read will still be reported upon read() + and/or next(). + """ + self._producer = [] + + def __iter__(self): + return self + + def unget(self, bytes): + """ + Places bytes back onto the front of the lazy stream. + + Future calls to read() will return those bytes first. The + stream position and thus tell() will be rewound. + """ + self.position -= len(bytes) + self._leftover = ''.join([bytes, self._leftover]) + + def exhaust(self): + """ + Exhausts the entire underlying stream. + + Useful for skipping and advancing sections. + """ + for thing in self: + pass + + +class ChunkIter(object): + def __init__(self, flo, chunk_size=1024**2): + self.flo = flo + self.chunk_size = chunk_size + + def next(self): + try: + data = self.flo.read(self.chunk_size) + except InputStreamExhausted: + raise StopIteration + if data: + return data + else: + raise StopIteration + + def __iter__(self): + return self + + +class LimitBytes(object): + """ Limit bytes for a file object. """ + def __init__(self, fileobject, length): + self._file = fileobject + self.remaining = length + + def read(self, num_bytes=None): + """ + Read data from the underlying file. + If you ask for too much or there isn't anything left, + this will raise an InputStreamExhausted error. + """ + if self.remaining <= 0: + raise InputStreamExhausted() + if num_bytes is None: + num_bytes = self.remaining + else: + num_bytes = min(num_bytes, self.remaining) + self.remaining -= num_bytes + return self._file.read(num_bytes) + + def exhaust(self): + """ + Exhaust this file until all of the bytes it was limited by + have been read. + """ + while self.remaining > 0: + num_bytes = min(self.remaining, 16384) + __ = self._file.read(num_bytes) + self.remaining -= num_bytes + + +class InterBoundaryIter(object): + """ + A Producer that will iterate over boundaries. + """ + def __init__(self, stream, boundary): + self._stream = stream + self._boundary = boundary + + def __iter__(self): + return self + + def next(self): + try: + return LazyStream(BoundaryIter(self._stream, self._boundary)) + except InputStreamExhausted: + raise StopIteration + +class BoundaryIter(object): + """ + A Producer that is sensitive to boundaries. + + Will happily yield bytes until a boundary is found. Will yield the + bytes before the boundary, throw away the boundary bytes + themselves, and push the post-boundary bytes back on the stream. + + The future calls to .next() after locating the boundary will raise + a StopIteration exception. + """ + def __init__(self, stream, boundary): + self._stream = stream + self._boundary = boundary + self._done = False + # rollback an additional six bytes because the format is like + # this: CRLF[--CRLF] + self._rollback = len(boundary) + 6 + + # Try to use mx fast string search if available. Otherwise + # use Python find. Wrap the latter for consistency. + unused_char = self._stream.read(1) + if not unused_char: + raise InputStreamExhausted + self._stream.unget(unused_char) + try: + from mx.TextTools import FS + self._fs = FS(boundary).find + except ImportError: + self._fs = lambda data: data.find(boundary) + + def __iter__(self): + return self + + def next(self): + if self._done: + raise StopIteration + + stream = self._stream + rollback = self._rollback + + bytes_read = 0 + chunks = [] + for bytes in stream: + bytes_read += len(bytes) + chunks.append(bytes) + if bytes_read > rollback: + break + if not bytes: + break + else: + self._done = True + + if not chunks: + raise StopIteration + + chunk = ''.join(chunks) + + boundary = self._find_boundary(chunk, len(chunk) < self._rollback) + + + if boundary: + end, next = boundary + stream.unget(chunk[next:]) + self._done = True + return chunk[:end] + else: + # make sure we dont treat a partial boundary (and + # its separators) as data + if not chunk[:-rollback]:# and len(chunk) >= (len(self._boundary) + 6): + # There's nothing left, we should just return and mark as done. + self._done = True + return chunk + else: + stream.unget(chunk[-rollback:]) + return chunk[:-rollback] + + def _find_boundary(self, data, eof = False): + """ + Finds a multipart boundary in data. + + Should no boundry exist in the data None is returned + instead. Otherwise a tuple containing + the indices of the following are returned: + + * the end of current encapsulation + + * the start of the next encapsulation + """ + index = self._fs(data) + if index < 0: + return None + else: + end = index + next = index + len(self._boundary) + data_len = len(data) - 1 + # backup over CRLF + if data[max(0,end-1)] == '\n': end -= 1 + if data[max(0,end-1)] == '\r': end -= 1 + # skip over --CRLF + if data[min(data_len,next)] == '-': next += 1 + if data[min(data_len,next)] == '-': next += 1 + if data[min(data_len,next)] == '\r': next += 1 + if data[min(data_len,next)] == '\n': next += 1 + return end, next + +def ParseBoundaryStream(stream, max_header_size): + """ + Parses one and exactly one stream that encapsulates a boundary. + """ + # Stream at beginning of header, look for end of header + # and parse it if found. The header must fit within one + # chunk. + chunk = stream.read(max_header_size) + # 'find' returns the top of these four bytes, so we'll + # need to munch them later to prevent them from polluting + # the payload. + header_end = chunk.find('\r\n\r\n') + + def parse_header(line): + from cgi import parse_header + main_value_pair, params = parse_header(line) + try: + name, value = main_value_pair.split(':', 1) + except: + raise ValueError("Invalid header: %r" % line) + return name, (value, params) + + if header_end == -1: + # we find no header, so we just mark this fact and pass on + # the stream verbatim + stream.unget(chunk) + return ('RAW', {}, stream) + + header = chunk[:header_end] + + # here we place any excess chunk back onto the stream, as + # well as throwing away the CRLFCRLF bytes from above. + stream.unget(chunk[header_end + 4:]) + + is_file_field = False + outdict = {} + + # eliminate blank lines + for line in header.split('\r\n'): + # This terminology ("main value" and "dictionary of + # parameters") is from the Python docs. + name, (value, params) = parse_header(line) + if name == 'content-disposition' and params.get('filename'): + is_file_field = True + + outdict[name] = value, params + + if is_file_field: + return ('FILE', outdict, stream) + else: + return ('FIELD', outdict, stream) + + +class Parser(object): + def __init__(self, stream, boundary): + self._stream = stream + self._separator = '--' + boundary + + def __iter__(self): + + boundarystream = InterBoundaryIter(self._stream, + self._separator) + + for sub_stream in boundarystream: + # Iterate over each part + yield ParseBoundaryStream(sub_stream, 1024) + + + Index: django/http/__init__.py =================================================================== --- django/http/__init__.py (revision 7354) +++ django/http/__init__.py (working copy) @@ -11,7 +11,7 @@ from django.utils.datastructures import MultiValueDict, FileDict from django.utils.encoding import smart_str, iri_to_uri, force_unicode - +from django.http.multipartparser import MultiPartParser from utils import * RESERVED_CHARS="!*'();:@&=+$,/?%#[]" @@ -30,6 +30,7 @@ self.GET, self.POST, self.COOKIES, self.META, self.FILES = {}, {}, {}, {}, {} self.path = '' self.method = None + self._upload_handlers = [] def __repr__(self): return '' % \ @@ -102,40 +103,35 @@ encoding = property(_get_encoding, _set_encoding) -def parse_file_upload(header_dict, post_data): - """Returns a tuple of (POST QueryDict, FILES MultiValueDict).""" - import email, email.Message - from cgi import parse_header - raw_message = '\r\n'.join(['%s:%s' % pair for pair in header_dict.items()]) - raw_message += '\r\n\r\n' + post_data - msg = email.message_from_string(raw_message) - POST = QueryDict('', mutable=True) - FILES = MultiValueDict() - for submessage in msg.get_payload(): - if submessage and isinstance(submessage, email.Message.Message): - name_dict = parse_header(submessage['Content-Disposition'])[1] - # name_dict is something like {'name': 'file', 'filename': 'test.txt'} for file uploads - # or {'name': 'blah'} for POST fields - # We assume all uploaded files have a 'filename' set. - if 'filename' in name_dict: - assert type([]) != type(submessage.get_payload()), "Nested MIME messages are not supported" - if not name_dict['filename'].strip(): - continue - # IE submits the full path, so trim everything but the basename. - # (We can't use os.path.basename because that uses the server's - # directory separator, which may not be the same as the - # client's one.) - filename = name_dict['filename'][name_dict['filename'].rfind("\\")+1:] - FILES.appendlist(name_dict['name'], FileDict({ - 'filename': filename, - 'content-type': 'Content-Type' in submessage and submessage['Content-Type'] or None, - 'content': submessage.get_payload(), - })) - else: - POST.appendlist(name_dict['name'], submessage.get_payload()) - return POST, FILES + def _set_upload_handlers(self, upload_handlers): + """ + Set the upload handler to the new handler given in the parameter. + """ + if hasattr(self, '_files'): + raise AttributeError("You cannot set the upload handler after the upload has been processed.") + self._upload_handlers = upload_handlers + def _get_upload_handlers(self): + return self._upload_handlers + upload_handlers = property(_get_upload_handlers, _set_upload_handlers) + + + def parse_file_upload(self, META, post_data): + """Returns a tuple of (POST QueryDict, FILES MultiValueDict).""" + from django.core.files.fileuploadhandler import TemporaryFileUploadHandler, MemoryFileUploadHandler + if not self.upload_handlers: + # Order here is *very* important. + self.upload_handlers = (MemoryFileUploadHandler(), + TemporaryFileUploadHandler()) + else: + self.upload_handlers = tuple(self.upload_handlers) + + parser = MultiPartParser(META, post_data, self.upload_handlers, + self.encoding) + return parser.parse() + + class QueryDict(MultiValueDict): """ A specialized MultiValueDict that takes a query string when initialized. Index: django/oldforms/__init__.py =================================================================== --- django/oldforms/__init__.py (revision 7354) +++ django/oldforms/__init__.py (working copy) @@ -680,18 +680,23 @@ self.field_name, self.is_required = field_name, is_required self.validator_list = [self.isNonEmptyFile] + validator_list - def isNonEmptyFile(self, field_data, all_data): - try: - content = field_data['content'] - except TypeError: - raise validators.CriticalValidationError, ugettext("No file was submitted. Check the encoding type on the form.") - if not content: + def isNonEmptyFile(self, new_data, all_data): + if hasattr(new_data, 'upload_errors'): + upload_errors = new_data.upload_errors() + if upload_errors: + raise validators.CriticalValidationError, upload_errors + if not new_data.file_size: raise validators.CriticalValidationError, ugettext("The submitted file is empty.") def render(self, data): return mark_safe(u'' % \ (self.get_id(), self.__class__.__name__, self.field_name)) + def prepare(self, new_data): + if hasattr(new_data, 'upload_errors'): + upload_errors = new_data.upload_errors() + new_data[self.field_name] = { '_file_upload_error': upload_errors } + def html2python(data): if data is None: raise EmptyValue Index: django/db/models/base.py =================================================================== --- django/db/models/base.py (revision 7354) +++ django/db/models/base.py (working copy) @@ -13,6 +13,7 @@ from django.utils.datastructures import SortedDict from django.utils.functional import curry from django.utils.encoding import smart_str, force_unicode, smart_unicode +from django.core.files.filemove import file_move_safe from django.conf import settings from itertools import izip import types @@ -384,12 +385,16 @@ def _get_FIELD_size(self, field): return os.path.getsize(self._get_FIELD_filename(field)) - def _save_FIELD_file(self, field, filename, raw_contents, save=True): + def _save_FIELD_file(self, field, filename, raw_field, save=True): directory = field.get_directory_name() try: # Create the date-based directory if it doesn't exist. os.makedirs(os.path.join(settings.MEDIA_ROOT, directory)) except OSError: # Directory probably already exists. pass + + if filename is None: + filename = raw_field.file_name + filename = field.get_filename(filename) # If the filename already exists, keep adding an underscore to the name of @@ -406,9 +411,18 @@ setattr(self, field.attname, filename) full_filename = self._get_FIELD_filename(field) - fp = open(full_filename, 'wb') - fp.write(raw_contents) - fp.close() + if hasattr(raw_field, 'temporary_file_path'): + raw_field.close() + file_move_safe(raw_field.temporary_file_path(), full_filename) + else: + from django.core.files import filelocks + fp = open(full_filename, 'wb') + # exclusive lock + filelocks.lock(fp, filelocks.LOCK_EX) + # Stream it into the file, from where it is. + for chunk in raw_field.chunk(65535): + fp.write(chunk) + fp.close() # Save the width and/or height, if applicable. if isinstance(field, ImageField) and (field.width_field or field.height_field): Index: django/db/models/fields/__init__.py =================================================================== --- django/db/models/fields/__init__.py (revision 7354) +++ django/db/models/fields/__init__.py (working copy) @@ -785,7 +785,8 @@ setattr(cls, 'get_%s_filename' % self.name, curry(cls._get_FIELD_filename, field=self)) setattr(cls, 'get_%s_url' % self.name, curry(cls._get_FIELD_url, field=self)) setattr(cls, 'get_%s_size' % self.name, curry(cls._get_FIELD_size, field=self)) - setattr(cls, 'save_%s_file' % self.name, lambda instance, filename, raw_contents, save=True: instance._save_FIELD_file(self, filename, raw_contents, save)) + setattr(cls, 'save_%s_file' % self.name, lambda instance, filename, raw_field, save=True: instance._save_FIELD_file(self, filename, raw_field, save)) + setattr(cls, 'move_%s_file' % self.name, lambda instance, raw_field, save=True: instance._save_FIELD_file(self, None, raw_field, save)) dispatcher.connect(self.delete_file, signal=signals.post_delete, sender=cls) def delete_file(self, instance): @@ -808,9 +809,9 @@ if new_data.get(upload_field_name, False): func = getattr(new_object, 'save_%s_file' % self.name) if rel: - func(new_data[upload_field_name][0]["filename"], new_data[upload_field_name][0]["content"], save) + func(new_data[upload_field_name][0].file_name, new_data[upload_field_name][0], save) else: - func(new_data[upload_field_name]["filename"], new_data[upload_field_name]["content"], save) + func(new_data[upload_field_name].file_name, new_data[upload_field_name], save) def get_directory_name(self): return os.path.normpath(force_unicode(datetime.datetime.now().strftime(smart_str(self.upload_to)))) @@ -823,7 +824,7 @@ def save_form_data(self, instance, data): from django.newforms.fields import UploadedFile if data and isinstance(data, UploadedFile): - getattr(instance, "save_%s_file" % self.name)(data.filename, data.content, save=False) + getattr(instance, "save_%s_file" % self.name)(data.filename, data.data, save=False) def formfield(self, **kwargs): defaults = {'form_class': forms.FileField} Index: django/conf/global_settings.py =================================================================== --- django/conf/global_settings.py (revision 7354) +++ django/conf/global_settings.py (working copy) @@ -224,6 +224,11 @@ # Example: "http://media.lawrence.com" MEDIA_URL = '' +# Directory to upload streamed files temporarily. +# A value of `None` means that it will use the default temporary +# directory for the server's operating system. +FILE_UPLOAD_TEMP_DIR = None + # Default formatting for date objects. See all available format strings here: # http://www.djangoproject.com/documentation/templates/#now DATE_FORMAT = 'N j, Y' Index: django/core/handlers/wsgi.py =================================================================== --- django/core/handlers/wsgi.py (revision 7354) +++ django/core/handlers/wsgi.py (working copy) @@ -78,6 +78,7 @@ self.path = force_unicode(environ['PATH_INFO']) self.META = environ self.method = environ['REQUEST_METHOD'].upper() + self._upload_handlers = [] def __repr__(self): # Since this is called as part of error handling, we need to be very @@ -112,9 +113,8 @@ # Populates self._post and self._files if self.method == 'POST': if self.environ.get('CONTENT_TYPE', '').startswith('multipart'): - header_dict = dict([(k, v) for k, v in self.environ.items() if k.startswith('HTTP_')]) - header_dict['Content-Type'] = self.environ.get('CONTENT_TYPE', '') - self._post, self._files = http.parse_file_upload(header_dict, self.raw_post_data) + self._raw_post_data = '' + self._post, self._files = self.parse_file_upload(self.META, self.environ['wsgi.input']) else: self._post, self._files = http.QueryDict(self.raw_post_data, encoding=self._encoding), datastructures.MultiValueDict() else: Index: django/core/handlers/modpython.py =================================================================== --- django/core/handlers/modpython.py (revision 7354) +++ django/core/handlers/modpython.py (working copy) @@ -16,6 +16,7 @@ def __init__(self, req): self._req = req self.path = force_unicode(req.uri) + self._upload_handlers = [] def __repr__(self): # Since this is called as part of error handling, we need to be very @@ -53,7 +54,8 @@ def _load_post_and_files(self): "Populates self._post and self._files" if 'content-type' in self._req.headers_in and self._req.headers_in['content-type'].startswith('multipart'): - self._post, self._files = http.parse_file_upload(self._req.headers_in, self.raw_post_data) + self._raw_post_data = '' + self._post, self._files = self.parse_file_upload(self.META, self._req) else: self._post, self._files = http.QueryDict(self.raw_post_data, encoding=self._encoding), datastructures.MultiValueDict() Index: django/core/files/filelocks.py =================================================================== --- django/core/files/filelocks.py (revision 0) +++ django/core/files/filelocks.py (revision 0) @@ -0,0 +1,68 @@ +""" +Locking portability based partially on example by +Jonathan Feignberg in python cookbook. + +Example Usage:: + + from django.utils import file_locks + + f = open('./file', 'wb') + + file_locks.lock(f, file_locks.LOCK_EX) + f.write('Django') + f.close() +""" + +__all__ = ('LOCK_EX','LOCK_SH','LOCK_NB','lock','unlock') + +system_type = None + +try: + import win32con + import win32file + import pywintypes + LOCK_EX = win32con.LOCKFILE_EXCLUSIVE_LOCK + LOCK_SH = 0 + LOCK_NB = win32con.LOCKFILE_FAIL_IMMEDIATELY + __overlapped = pywintypes.OVERLAPPED() + system_type = 'nt' +except (ImportError, AttributeError): + pass + +try: + import fcntl + LOCK_EX = fcntl.LOCK_EX + LOCK_SH = fcntl.LOCK_SH + LOCK_NB = fcntl.LOCK_NB + system_type = 'posix' +except (ImportError, AttributeError): + pass + + + +if system_type == 'nt': + def lock(file, flags): + hfile = win32file._get_osfhandle(file.fileno()) + win32file.LockFileEx(hfile, flags, 0, -0x10000, __overlapped) + + def unlock(file): + hfile = win32file._get_osfhandle(file.fileno()) + win32file.UnlockFileEx(hfile, 0, -0x10000, __overlapped) + +elif system_type =='posix': + def lock(file, flags): + fcntl.flock(file.fileno(), flags) + + def unlock(file): + fcntl.flock(file.fileno(), fcntl.LOCK_UN) + +else: + # File locking is not supported. + LOCK_EX = LOCK_SH = LOCK_NB = None + + # Dummy functions that don't do anything. + def lock(file, flags): + pass + + def unlock(file): + pass Index: django/core/files/uploadedfile.py =================================================================== --- django/core/files/uploadedfile.py (revision 0) +++ django/core/files/uploadedfile.py (revision 0) @@ -0,0 +1,191 @@ +""" +The uploaded file objects for Django. +This contains the base UploadedFile and the TemporaryUploadedFile +derived class. +""" + +__all__ = ('UploadedFile', 'TemporaryUploadedFile', 'InMemoryUploadedFile') + +class UploadedFile(object): + """ + The UploadedFile object behaves somewhat like a file + object and represents some data that the user submitted + and is stored in some form. + """ + DEFAULT_CHUNK_SIZE = 64 * 2**10 + + def __init__(self): + self.file_size = None + self.file_name = None + self.content_type = None + self.charset = None + pass + + def file_size(self): + return self.file_size + + def chunk(self, chunk_size=None): + """ + Read the file to generate chunks of chunk_size bytes. + """ + if not chunk_size: + chunk_size = UploadedFile.DEFAULT_CHUNK_SIZE + + if hasattr(self, 'seek'): + self.seek(0) + # Assume the pointer is at zero... + counter = self.file_size() + + while counter > 0: + yield self.read(chunk_size) + counter -= chunk_size + + + def multiple_chunks(self, chunk_size=None): + """ + Return True if you can expect multiple chunks, False otherwise. + Note: If a particular file representation is in memory, then + override this to return False. + """ + if not chunk_size: + chunk_size = UploadedFile.DEFAULT_CHUNK_SIZE + return self.file_size() < chunk_size + + + def read(self, num_bytes=None): + """ + Read from the file in whatever representation it has. + """ + raise NotImplementedError() + + def open(self): + """ + Open the file, if one needs to. + """ + pass + + + def close(self): + """ + Close the file, if one needs to. + """ + pass + + def __getitem__(self, key): + """ + This maintains backwards compatibility. + """ + import warnings + warnings.warn("The dictionary access of uploaded file objects is deprecated. Use the new object interface instead.", DeprecationWarning) + # Dictionary to translate labels + # for backwards compatbility. + # Should be removed at some point. + backwards_translate = { + 'filename': 'file_name', + 'content-type': 'content_type', + } + + if key == 'content': + return self.read() + else: + return getattr(self, backwards_translate.get(key, key)) + + def __repr__(self): + """ + This representation could be anything and can be overridden. + This is mostly done to make it look somewhat useful. + """ + _dict = { + 'file_name': self.file_name, + 'content_type': self.content_type, + 'content': '', + } + return repr(_dict) + + +class TemporaryUploadedFile(UploadedFile): + """ + Upload a file to a temporary file. + """ + + def __init__(self, file, file_name, content_type, file_size, charset): + self.file = file + self.file_name = file_name + self.path = file.name + self.content_type = content_type + self.file_size = file_size + self.charset = charset + self.file.seek(0) + + def temporary_file_path(self): + """ + Return the full path of this file. + """ + return self.path + + def read(self, *args, **kwargs): + return self.file.read(*args, **kwargs) + + def open(self): + """ + Assume the person meant to seek. + """ + self.seek(0) + + def seek(self, *args, **kwargs): + self.file.seek(*args, **kwargs) + + +class InMemoryUploadedFile(UploadedFile): + """ + Upload a file into memory. + """ + def __init__(self, file, field_name, file_name, content_type, charset): + self.file = file + self.field_name = field_name + self.file_name = file_name + self.content_type = content_type + self.charset = charset + self.file.seek(0) + + def seek(self, *args, **kwargs): + self.file.seek(*args, **kwargs) + + def open(self): + self.seek(0) + + def read(self, *args, **kwargs): + return self.file.read(*args, **kwargs) + + def chunk(self, chunk_size=None): + """ + Return the entirety of the data regardless. + """ + self.file.seek(0) + return self.read() + + def multiple_chunks(self, chunk_size=None): + """ + Since it's in memory, we'll never have multiple chunks. + """ + return False + + +class SimpleUploadedFile(InMemoryUploadedFile): + """ + A simple representation of a file, which + just has content, size, and a name. + """ + def __init__(self, name, content, content_type='text/plain'): + try: + from cStringIO import StringIO + except ImportError: + from StringIO import StringIO + self.file = StringIO(content or '') + self.file_name = name + self.field_name = None + self.file_size = len(content or '') + self.content_type = content_type + self.charset = None + self.file.seek(0) + Index: django/core/files/__init__.py =================================================================== --- django/core/files/__init__.py (revision 0) +++ django/core/files/__init__.py (revision 0) @@ -0,0 +1 @@ + Index: django/core/files/fileuploadhandler.py =================================================================== --- django/core/files/fileuploadhandler.py (revision 0) +++ django/core/files/fileuploadhandler.py (revision 0) @@ -0,0 +1,243 @@ +""" A fileuploadhandler base and default subclass for handling file uploads. +""" +import os +try: + from cStringIO import StringIO +except ImportError: + from StringIO import StringIO + +from django.utils.encoding import force_unicode +from django.utils.datastructures import MultiValueDict + +from django.core.files.uploadedfile import TemporaryUploadedFile, InMemoryUploadedFile + +__all__ = ('UploadFileException','StopUpload', 'SkipFile', + 'FileUploadHandler', 'TemporaryFileUploadHandler', + 'MemoryFileUploadHandler') + + +class UploadFileException(Exception): + """ Any error having to do with Uploading Files. """ + pass + +class StopUpload(UploadFileException): + """ This exception is raised when an upload must abort. """ + pass + +class SkipFile(UploadFileException): + """ This exception is raised when a file needs to be skipped. """ + pass + + +class FileUploadHandler(object): + """ FileUploadHandler will take data and handle file uploads + in a streamed fashion. + """ + chunk_size = 64 * 2 ** 10 #: The default chunk size is 64 KB. + + def __init__(self): + " Initialize some local variables. " + self.file_name = None + self.content_type = None + self.content_length = None + self.charset = None + + def handle_raw_input(self, input_data, META, content_length, boundary, encoding=None): + """ + Handle the raw input from the client. + Parameters: + *input_data* -- An object that supports reading via .read(). + *content_length* -- The (integer) value of the Content-Length header from the client. + *boundary* -- The boundary from the Content-Type header. Be sure to prepend two '--'. + """ + pass + + def new_file(self, field_name, file_name, content_type, content_length, charset=None): + """ + Signal that a new file has been started. + + Warning: Do not trust content_length, if you get it at all. + """ + self.field_name = field_name + self.file_name = file_name + self.content_type = content_type + self.content_length = content_length + self.charset = charset + + def receive_data_chunk(self, raw_data, start, stop): + """ + Receive data from the streamed upload parser. + Start and stop are the positions in the file. + This equality should always be true:: + len(raw_data) = stop - start + """ + raise NotImplementedError() + + def file_complete(self, file_size): + """ + Signal that a file has completed. + File size corresponds to the actual size accumulated + by all the chunks. + + This should return a valid UploadedFile object. + """ + raise NotImplementedError() + + def upload_complete(self): + """ + Signal that the upload is complete. + Do any cleanup that is necessary for this handler. + """ + pass + + + +class TemporaryFileUploadHandler(FileUploadHandler): + """ + Upload the streaming data into a temporary file. + """ + def __init__(self, *args, **kwargs): + """ Import settings for later. """ + super(TemporaryFileUploadHandler, self).__init__(*args, **kwargs) + global settings + from django.conf import settings + + def new_file(self, file_name, *args, **kwargs): + """ + Create the file object to append to as data is coming in. + """ + super(TemporaryFileUploadHandler, self).new_file(file_name, *args, **kwargs) + self.file = TemporaryFile(settings.FILE_UPLOAD_TEMP_DIR) + self.write = self.file.write + + def receive_data_chunk(self, raw_data, start, stop): + """ + Once we get the data, we will save it to our file. + """ + self.write(raw_data) + + def file_complete(self, file_size): + """ + Signal that a file has completed. + File size corresponds to the actual size accumulated + by all the chunks. + + This should return a valid UploadedFile object. + """ + self.file.seek(0) + return TemporaryUploadedFile(self.file, self.file_name, + self.content_type, file_size, + self.charset) + + +class TemporaryFile(object): + """ + A temporary file that tries to delete itself when garbage collected. + """ + def __init__(self, dir): + import tempfile + if not dir: + dir = tempfile.gettempdir() + try: + (fd, name) = tempfile.mkstemp(suffix='.upload', dir=dir) + self.file = os.fdopen(fd, 'w+b') + except (OSError, IOError): + raise OSError, "Could not create temporary file for uploading, have you set settings.FILE_UPLOAD_TEMP_DIR correctly?" + self.name = name + + def __getattr__(self, name): + a = getattr(self.__dict__['file'], name) + if type(a) != type(0): + setattr(self, name, a) + return a + + def __del__(self): + try: + os.unlink(self.name) + except OSError: + pass + + +class MemoryFileUploadHandler(FileUploadHandler): + """ + The MemoryFileUploadHandler will place the data directly into memory. + """ + + def __init__(self): + pass + + + def handle_raw_input(self, input_data, META, content_length, boundary, encoding=None): + """ + Parse the input data in-memory. + """ + if content_length > 2621440: + # If the post is greater than 2.5 MB, do nothing. + return + + from django.http import QueryDict + import email, email.Message + from cgi import parse_header + + ##### + # Get the headers from the META information. + headers = [] + if 'HTTP_CONTENT_TYPE' not in META: + headers.append('Content-Type: %s' % (META.get('CONTENT_TYPE', ''))) + + if 'HTTP_CONTENT_LENGTH' not in META: + headers.append('Content-Length: %s' % (META.get('CONTENT_LENGTH', '0'))) + + for key, value in META.items(): + if key.startswith('HTTP_'): + headers.append('%s: %s' % (key[5:].replace('_','-').title(), value)) + + raw_message = '\r\n'.join(headers) + raw_message += '\r\n\r\n' + input_data.read() + + msg = email.message_from_string(raw_message) + POST = QueryDict('', mutable=True) + FILES = MultiValueDict() + for submessage in msg.get_payload(): + if submessage and isinstance(submessage, email.Message.Message): + name_dict = parse_header(submessage['Content-Disposition'])[1] + field_name = force_unicode(name_dict['name'], encoding, errors='replace') + + if 'filename' in name_dict: + assert not isinstance(submessage.get_payload(), list), "Nested MIME messages are not supported" + if not name_dict['filename'].strip(): + continue + + filename = force_unicode(name_dict['filename'][name_dict['filename'].rfind("\\")+1:], + encoding, errors='replace') + content_type = 'Content-Type' in submessage and submessage['Content-Type'] or None + + file_obj = InMemoryUploadedFile(StringIO(submessage.get_payload()), + field_name, filename, content_type, None) + + FILES.appendlist(field_name, file_obj) + else: + content = force_unicode(submessage.get_payload(), encoding, errors='replace') + POST.appendlist(field_name, content) + + return POST, FILES + + + def new_file(self, field_name, file_name, content_type, content_length, charset): + """ + Do Nothing. + """ + return + + def receive_data_chunk(self, raw_data, start, stop): + """ + Do nothing. + """ + return raw_data + + def file_complete(self, file_size): + """ + Do nothing. + """ + return + Index: django/core/files/filemove.py =================================================================== --- django/core/files/filemove.py (revision 0) +++ django/core/files/filemove.py (revision 0) @@ -0,0 +1,52 @@ +import os + +__all__ = ('file_move_safe',) + +try: + import shutil + file_move = shutil.move +except ImportError: + file_move = os.rename + +def file_move_safe(old_file_name, new_file_name, chunk_size = 1024*64, allow_overwrite=False): + """ + Moves a file from one location to another in the safest way possible. + + First, it tries using shutils.move, which is OS-dependent but doesn't + break with change of filesystems. Then it tries os.rename, which will + break if it encounters a change in filesystems. Lastly, it streams + it manually from one file to another in python. + + Without ``allow_overwrite``, if the destination file exists, the + file will raise an IOError. + """ + + from django.core.files import filelocks + + if old_file_name == new_file_name: + # No file moving takes place. + return + + if not allow_overwrite and os.path.exists(new_file_name): + raise IOError, "Django does not allow overwriting files." + + try: + file_move(old_file_name, new_file_name) + return + except OSError: # moving to another filesystem + pass + + new_file = open(new_file_name, 'wb') + # exclusive lock + filelocks.lock(new_file, filelocks.LOCK_EX) + old_file = open(old_file_name, 'rb') + current_chunk = None + + while current_chunk != '': + current_chunk = old_file.read(chunk_size) + new_file.write(current_chunk) + + new_file.close() + old_file.close() + + os.remove(old_file_name) Index: django/newforms/fields.py =================================================================== --- django/newforms/fields.py (revision 7354) +++ django/newforms/fields.py (working copy) @@ -416,9 +416,9 @@ class UploadedFile(StrAndUnicode): "A wrapper for files uploaded in a FileField" - def __init__(self, filename, content): + def __init__(self, filename, data): self.filename = filename - self.content = content + self.data = data def __unicode__(self): """ @@ -445,12 +445,10 @@ elif not data and initial: return initial try: - f = UploadedFile(data['filename'], data['content']) - except TypeError: + f = UploadedFile(data.file_name, data) + except (TypeError, AttributeError): raise ValidationError(self.error_messages['invalid']) - except KeyError: - raise ValidationError(self.error_messages['missing']) - if not f.content: + if not f.data.file_size: raise ValidationError(self.error_messages['empty']) return f @@ -470,15 +468,26 @@ elif not data and initial: return initial from PIL import Image - from cStringIO import StringIO + + # We need to get the file, it either has a path + # or we have to read it all into memory... + if hasattr(data, 'temporary_file_path'): + file = data.temporary_file_path() + else: + try: + from cStringIO import StringIO + except ImportError: + from StringIO import StringIO + file = StringIO(data.read()) + try: # load() is the only method that can spot a truncated JPEG, # but it cannot be called sanely after verify() - trial_image = Image.open(StringIO(f.content)) + trial_image = Image.open(file) trial_image.load() # verify() is the only method that can spot a corrupt PNG, # but it must be called immediately after the constructor - trial_image = Image.open(StringIO(f.content)) + trial_image = Image.open(file) trial_image.verify() except Exception: # Python Imaging Library doesn't recognize it as an image raise ValidationError(self.error_messages['invalid_image']) Index: tests/modeltests/model_forms/models.py =================================================================== --- tests/modeltests/model_forms/models.py (revision 7354) +++ tests/modeltests/model_forms/models.py (working copy) @@ -75,6 +75,7 @@ __test__ = {'API_TESTS': """ >>> from django import newforms as forms >>> from django.newforms.models import ModelForm +>>> from django.core.files.uploadedfile import SimpleUploadedFile The bare bones, absolutely nothing custom, basic case. @@ -792,7 +793,7 @@ # Upload a file and ensure it all works as expected. ->>> f = TextFileForm(data={'description': u'Assistance'}, files={'file': {'filename': 'test1.txt', 'content': 'hello world'}}) +>>> f = TextFileForm(data={'description': u'Assistance'}, files={'file': SimpleUploadedFile('test1.txt', 'hello world')}) >>> f.is_valid() True >>> type(f.cleaned_data['file']) @@ -819,7 +820,7 @@ # Override the file by uploading a new one. ->>> f = TextFileForm(data={'description': u'Assistance'}, files={'file': {'filename': 'test2.txt', 'content': 'hello world'}}, instance=instance) +>>> f = TextFileForm(data={'description': u'Assistance'}, files={'file': SimpleUploadedFile('test2.txt', 'hello world')}, instance=instance) >>> f.is_valid() True >>> instance = f.save() @@ -838,7 +839,7 @@ >>> instance.file '' ->>> f = TextFileForm(data={'description': u'Assistance'}, files={'file': {'filename': 'test3.txt', 'content': 'hello world'}}, instance=instance) +>>> f = TextFileForm(data={'description': u'Assistance'}, files={'file': SimpleUploadedFile('test3.txt', 'hello world')}, instance=instance) >>> f.is_valid() True >>> instance = f.save() @@ -858,7 +859,7 @@ >>> image_data = open(os.path.join(os.path.dirname(__file__), "test.png")).read() ->>> f = ImageFileForm(data={'description': u'An image'}, files={'image': {'filename': 'test.png', 'content': image_data}}) +>>> f = ImageFileForm(data={'description': u'An image'}, files={'image': SimpleUploadedFile('test.png', image_data)}) >>> f.is_valid() True >>> type(f.cleaned_data['image']) @@ -885,7 +886,7 @@ # Override the file by uploading a new one. ->>> f = ImageFileForm(data={'description': u'Changed it'}, files={'image': {'filename': 'test2.png', 'content': image_data}}, instance=instance) +>>> f = ImageFileForm(data={'description': u'Changed it'}, files={'image': SimpleUploadedFile('test2.png', image_data)}, instance=instance) >>> f.is_valid() True >>> instance = f.save() @@ -904,7 +905,7 @@ >>> instance.image '' ->>> f = ImageFileForm(data={'description': u'And a final one'}, files={'image': {'filename': 'test3.png', 'content': image_data}}, instance=instance) +>>> f = ImageFileForm(data={'description': u'And a final one'}, files={'image': SimpleUploadedFile('test3.png', image_data)}, instance=instance) >>> f.is_valid() True >>> instance = f.save() Index: tests/regressiontests/bug639/tests.py =================================================================== --- tests/regressiontests/bug639/tests.py (revision 7354) +++ tests/regressiontests/bug639/tests.py (working copy) @@ -9,6 +9,7 @@ from regressiontests.bug639.models import Photo from django.http import QueryDict from django.utils.datastructures import MultiValueDict +from django.core.files.uploadedfile import SimpleUploadedFile class Bug639Test(unittest.TestCase): @@ -21,11 +22,7 @@ # Fake a request query dict with the file qd = QueryDict("title=Testing&image=", mutable=True) - qd["image_file"] = { - "filename" : "test.jpg", - "content-type" : "image/jpeg", - "content" : img - } + qd["image_file"] = SimpleUploadedFile('test.jpg', img, 'image/jpeg') manip = Photo.AddManipulator() manip.do_html2python(qd) @@ -39,4 +36,4 @@ Make sure to delete the "uploaded" file to avoid clogging /tmp. """ p = Photo.objects.get() - os.unlink(p.get_image_filename()) \ No newline at end of file + os.unlink(p.get_image_filename()) Index: tests/regressiontests/forms/error_messages.py =================================================================== --- tests/regressiontests/forms/error_messages.py (revision 7354) +++ tests/regressiontests/forms/error_messages.py (working copy) @@ -1,6 +1,7 @@ # -*- coding: utf-8 -*- tests = r""" >>> from django.newforms import * +>>> from django.core.files.uploadedfile import SimpleUploadedFile # CharField ################################################################### @@ -214,11 +215,11 @@ Traceback (most recent call last): ... ValidationError: [u'INVALID'] ->>> f.clean({}) +>>> f.clean(SimpleUploadedFile('name', None)) Traceback (most recent call last): ... -ValidationError: [u'MISSING'] ->>> f.clean({'filename': 'name', 'content':''}) +ValidationError: [u'EMPTY FILE'] +>>> f.clean(SimpleUploadedFile('name', '')) Traceback (most recent call last): ... ValidationError: [u'EMPTY FILE'] Index: tests/regressiontests/forms/fields.py =================================================================== --- tests/regressiontests/forms/fields.py (revision 7354) +++ tests/regressiontests/forms/fields.py (working copy) @@ -2,6 +2,7 @@ tests = r""" >>> from django.newforms import * >>> from django.newforms.widgets import RadioFieldRenderer +>>> from django.core.files.uploadedfile import SimpleUploadedFile >>> import datetime >>> import time >>> import re @@ -773,12 +774,12 @@ >>> f.clean({}) Traceback (most recent call last): ... -ValidationError: [u'No file was submitted.'] +ValidationError: [u'No file was submitted. Check the encoding type on the form.'] >>> f.clean({}, '') Traceback (most recent call last): ... -ValidationError: [u'No file was submitted.'] +ValidationError: [u'No file was submitted. Check the encoding type on the form.'] >>> f.clean({}, 'files/test3.pdf') 'files/test3.pdf' @@ -788,20 +789,20 @@ ... ValidationError: [u'No file was submitted. Check the encoding type on the form.'] ->>> f.clean({'filename': 'name', 'content': None}) +>>> f.clean(SimpleUploadedFile('name', None)) Traceback (most recent call last): ... ValidationError: [u'The submitted file is empty.'] ->>> f.clean({'filename': 'name', 'content': ''}) +>>> f.clean(SimpleUploadedFile('name', '')) Traceback (most recent call last): ... ValidationError: [u'The submitted file is empty.'] ->>> type(f.clean({'filename': 'name', 'content': 'Some File Content'})) +>>> type(f.clean(SimpleUploadedFile('name', 'Some File Content'))) ->>> type(f.clean({'filename': 'name', 'content': 'Some File Content'}, 'files/test4.pdf')) +>>> type(f.clean(SimpleUploadedFile('name', 'Some File Content'), 'files/test4.pdf')) # URLField ################################################################## Index: tests/regressiontests/forms/forms.py =================================================================== --- tests/regressiontests/forms/forms.py (revision 7354) +++ tests/regressiontests/forms/forms.py (working copy) @@ -1,6 +1,7 @@ # -*- coding: utf-8 -*- tests = r""" >>> from django.newforms import * +>>> from django.core.files.uploadedfile import SimpleUploadedFile >>> import datetime >>> import time >>> import re @@ -1465,7 +1466,7 @@ >>> print f File1:
  • This field is required.
->>> f = FileForm(data={}, files={'file1': {'filename': 'name', 'content':''}}, auto_id=False) +>>> f = FileForm(data={}, files={'file1': SimpleUploadedFile('name', '')}, auto_id=False) >>> print f File1:
  • The submitted file is empty.
@@ -1473,7 +1474,7 @@ >>> print f File1:
  • No file was submitted. Check the encoding type on the form.
->>> f = FileForm(data={}, files={'file1': {'filename': 'name', 'content':'some content'}}, auto_id=False) +>>> f = FileForm(data={}, files={'file1': SimpleUploadedFile('name', 'some content')}, auto_id=False) >>> print f File1: >>> f.is_valid() Index: tests/regressiontests/test_client_regress/views.py =================================================================== --- tests/regressiontests/test_client_regress/views.py (revision 7354) +++ tests/regressiontests/test_client_regress/views.py (working copy) @@ -1,5 +1,6 @@ from django.contrib.auth.decorators import login_required from django.http import HttpResponse, HttpResponseRedirect, HttpResponseServerError +import sha def no_template_view(request): "A simple view that expects a GET request, and returns a rendered template" @@ -10,13 +11,36 @@ Check that a file upload can be updated into the POST dictionary without going pear-shaped. """ + from django.core.files.uploadedfile import UploadedFile form_data = request.POST.copy() form_data.update(request.FILES) - if isinstance(form_data['file_field'], dict) and isinstance(form_data['name'], unicode): + if isinstance(form_data['file_field'], UploadedFile) and isinstance(form_data['name'], unicode): return HttpResponse('') else: return HttpResponseServerError() +def file_upload_view_verify(request): + """ + Use the sha digest hash to verify the uploaded contents. + """ + from django.core.files.uploadedfile import UploadedFile + form_data = request.POST.copy() + form_data.update(request.FILES) + for key, value in form_data.items(): + if key.endswith('_hash'): + continue + if key + '_hash' not in form_data: + continue + submitted_hash = form_data[key + '_hash'] + if isinstance(value, UploadedFile): + new_hash = sha.new(value.read()).hexdigest() + else: + new_hash = sha.new(value).hexdigest() + if new_hash != submitted_hash: + return HttpResponseServerError() + + return HttpResponse('') + def get_view(request): "A simple login protected view" return HttpResponse("Hello world") @@ -37,4 +61,4 @@ def login_protected_redirect_view(request): "A view that redirects all requests to the GET view" return HttpResponseRedirect('/test_client_regress/get_view/') -login_protected_redirect_view = login_required(login_protected_redirect_view) \ No newline at end of file +login_protected_redirect_view = login_required(login_protected_redirect_view) Index: tests/regressiontests/test_client_regress/models.py =================================================================== --- tests/regressiontests/test_client_regress/models.py (revision 7354) +++ tests/regressiontests/test_client_regress/models.py (working copy) @@ -5,6 +5,7 @@ from django.test import Client, TestCase from django.core.urlresolvers import reverse import os +import sha class AssertContainsTests(TestCase): def test_contains(self): @@ -243,6 +244,44 @@ response = self.client.post('/test_client_regress/file_upload/', post_data) self.assertEqual(response.status_code, 200) + def test_large_upload(self): + import tempfile + dir = tempfile.gettempdir() + + (fd, name1) = tempfile.mkstemp(suffix='.file1', dir=dir) + file1 = os.fdopen(fd, 'w+b') + file1.write('a' * (2 ** 21)) + file1.seek(0) + + (fd, name2) = tempfile.mkstemp(suffix='.file2', dir=dir) + file2 = os.fdopen(fd, 'w+b') + file2.write('a' * (10 * 2 ** 20)) + file2.seek(0) + + post_data = { + 'name': 'Ringo', + 'file_field1': file1, + 'file_field2': file2, + } + + for key in post_data.keys(): + try: + post_data[key + '_hash'] = sha.new(post_data[key].read()).hexdigest() + post_data[key].seek(0) + except AttributeError: + post_data[key + '_hash'] = sha.new(post_data[key]).hexdigest() + + response = self.client.post('/test_client_regress/file_upload_verify/', post_data) + + for name in (name1, name2): + try: + os.unlink(name) + except: + pass + + self.assertEqual(response.status_code, 200) + + class LoginTests(TestCase): fixtures = ['testdata'] Index: tests/regressiontests/test_client_regress/urls.py =================================================================== --- tests/regressiontests/test_client_regress/urls.py (revision 7354) +++ tests/regressiontests/test_client_regress/urls.py (working copy) @@ -4,6 +4,7 @@ urlpatterns = patterns('', (r'^no_template_view/$', views.no_template_view), (r'^file_upload/$', views.file_upload_view), + (r'^file_upload_verify/$', views.file_upload_view_verify), (r'^get_view/$', views.get_view), url(r'^arg_view/(?P.+)/$', views.view_with_argument, name='arg_view'), (r'^login_protected_redirect_view/$', views.login_protected_redirect_view) Index: AUTHORS =================================================================== --- AUTHORS (revision 7354) +++ AUTHORS (working copy) @@ -58,7 +58,7 @@ Jökull Sólberg Auðunsson Arthur David Avsajanishvili - axiak@mit.edu + Mike Axiak Niran Babalola Morten Bagai Mikaël Barbero @@ -135,6 +135,7 @@ Marc Fargas Szilveszter Farkas favo@exoweb.net + fdr Dmitri Fedortchenko Bill Fenner Stefane Fermgier Index: docs/settings.txt =================================================================== --- docs/settings.txt (revision 7354) +++ docs/settings.txt (working copy) @@ -513,6 +513,15 @@ The character encoding used to decode any files read from disk. This includes template files and initial SQL data files. +FILE_UPLOAD_TEMP_DIR +-------------------- + +**New in Django development version** + +Default: ``None`` + +The directory to store data temporarily while uploading files. If ``None``, Django will use the standard temporary directory for the operating system. For example, this will default to '/tmp' on *nix-style operating systems. + FIXTURE_DIRS -------------