Index: django/http/multipartparser.py =================================================================== --- django/http/multipartparser.py (revision 0) +++ django/http/multipartparser.py (revision 0) @@ -0,0 +1,599 @@ +""" +MultiPart parsing for file uploads. + +This object will take the file upload headers +and the file upload handler and chunk the upload +data for the handler to deal with. +""" +from django.utils.datastructures import MultiValueDict +from django.utils.encoding import force_unicode + +__all__ = ('MultiPartParser','MultiPartParserError','InputStreamExhausted') + +class MultiPartParserError(Exception): + pass + +class InputStreamExhausted(Exception): + """ No more reads are allowed from this device. """ + pass + +class MultiPartParser(object): + """ + A rfc2388 multipart/form-data parser. + + parse() reads the input stream in chunk_size chunks and returns a + tuple of (POST MultiValueDict, FILES MultiValueDict). If + file_upload_dir is defined files will be streamed to temporary + files in the specified directory. + """ + def __init__(self, META, input_data, upload_handlers, encoding=None): + """ + Initialize the MultiPartParser object. + + *META* -- The standard META dictionary in Django request objects. + *input_data* -- The raw post data, as a bytestring. + *upload_handler* -- An object of type UploadHandler + that performs operations on the uploaded + data. + *encoding* -- The encoding with which to treat the incoming data. + """ + # Import cgi utilities for (near) future use. + global parse_header, valid_boundary, settings + from django.conf import settings + from cgi import valid_boundary, parse_header + + # + # Content-Type should containt multipart and the boundary information. + # + + content_type = META.get('HTTP_CONTENT_TYPE', META.get('CONTENT_TYPE', '')) + if not content_type.startswith('multipart/'): + raise MultiPartParserError('Invalid Content-Type: %s' % + content_type) + + # Parse the header to get the boundary to split the parts. + ctypes, opts = parse_header(content_type) + boundary = opts.get('boundary') + if not boundary or not valid_boundary(boundary): + raise MultiPartParserError('Invalid boundary in multipart: %s' % + boundary) + + + # + # Content-Length should contain the length of the body we are about + # to receive. + # + try: + content_length = int(META.get('HTTP_CONTENT_LENGTH', + META.get('CONTENT_LENGTH',0))) + except (ValueError, TypeError): + # For now set it to 0...we'll try again later on down. + content_length = 0 + + if content_length <= 0: + # This means we shouldn't continue...raise an error. + raise MultiPartParserError("Invalid content length: %r" % content_length) + + self._boundary = boundary + self._input_data = input_data + + # For compatibility with low-level network APIs (with 32-bit integers), + # the chunk size should be < 2^31, but still divisible by 4. + self._chunk_size = min(2147483644, *[x.chunk_size for x in upload_handlers + if x.chunk_size]) + + self._meta = META + self._encoding = encoding or settings.DEFAULT_CHARSET + self._content_length = content_length + self._upload_handlers = upload_handlers + + def parse(self): + """ + Parse the POST data and break it into a FILES MultiValueDict + and a POST MultiValueDict. + + *returns* -- A tuple containing the POST and FILES dictionary, + respectively. + """ + from django.core.files.fileuploadhandler import StopUpload, SkipFile + from django.http import QueryDict + + encoding = self._encoding + handlers = self._upload_handlers + + limited_input_data = LimitBytes(self._input_data, self._content_length) + + # See if the handler will want to take care of the parsing. + # This allows overriding everything if somebody wants it. + for handler in handlers: + result = handler.handle_raw_input(limited_input_data, + self._meta, + self._content_length, + self._boundary, + encoding) + if result is not None: + return result[0], result[1] + + # Create the data structures to be used later. + self._post = QueryDict('', mutable=True) + self._files = MultiValueDict() + + # Instantiate the parser and stream: + stream = LazyStream(ChunkIter(limited_input_data, self._chunk_size)) + + # Whether or not to signal a file-completion at the beginning of the loop. + old_field_name = None + counters = [0] * len(handlers) + + for item_type, meta_data, stream in Parser(stream, self._boundary): + if old_field_name: + # We run this at the beginning of the next loop + # since we cannot be sure a file is complete until + # we hit the next boundary/part of the multipart content. + self.handle_file_complete(old_field_name, counters) + + try: + disposition = meta_data['content-disposition'][1] + field_name = disposition['name'].strip() + except (KeyError, IndexError, AttributeError): + continue + + transfer_encoding = meta_data.get('content-transfer-encoding') + field_name = force_unicode(field_name, encoding, errors='replace') + + if item_type == 'FIELD': + # This is a post field, we can just set it in the post + if transfer_encoding == 'base64': + raw_data = stream.read() + try: + data = str(raw_data).decode('base64') + except: + data = raw_data + else: + data = stream.read() + + self._post.appendlist(field_name, + force_unicode(data, encoding, errors='replace')) + elif item_type == 'FILE': + # This is a file, use the handler... + file_successful = True + file_name = self.IE_sanitize(disposition.get('filename')) + if not file_name: + continue + + file_name = force_unicode(file_name, encoding, errors='replace') + + content_type = meta_data.get('content-type', ('',))[0].strip() + try: + charset = meta_data.get('content-type', (0,{}))[1].get('charset', None) + except: + charset = None + + try: + content_length = int(meta_data.get('content-length')[0]) + except (IndexError, TypeError, ValueError): + content_length = None + + counters = [0] * len(handlers) + try: + for handler in handlers: + retval = handler.new_file(field_name, file_name, + content_type, content_length, + charset) + if retval: + break + + for chunk in stream: + if transfer_encoding == 'base64': + # We only special-case base64 transfer encoding + try: + chunk = str(chunk).decode('base64') + except Exception, e: + # Since this is only a chunk, any error is an unfixable error. + raise MultiValueParseError("Could not decode base64 data: %r" % e) + + for i, handler in enumerate(handlers): + chunk_length = len(chunk) + counters[i] += chunk_length + chunk = handler.receive_data_chunk(chunk, + counters[i] - chunk_length, + counters[i]) + if chunk is None: + # If the chunk received by the handler is None, then don't continue. + break + + except (StopUpload, SkipFile), e: + file_successful = False + if isinstance(e, SkipFile): + # Just use up the rest of this file... + stream.exhaust() + elif isinstance(e, StopUpload): + # Abort the parsing and break + parser.abort() + break + else: + # Handle file upload completions on next iteration. + old_field_name = field_name + else: + # If this is neither a FIELD or a FILE, just exhaust the stream. + stream.exhuast() + + # Make sure that the request data is all fed + limited_input_data.exhaust() + + # Signal that the upload has completed. + for handler in handlers: + retval = handler.upload_complete() + if retval: + break + + return self._post, self._files + + def handle_file_complete(self, old_field_name, counters): + """ + Handle all the signalling that takes place when a file is complete. + """ + for i, handler in enumerate(self._upload_handlers): + file_obj = handler.file_complete(counters[i]) + if file_obj: + # If it returns a file object, then set the files dict. + self._files.appendlist(force_unicode(old_field_name, + self._encoding, + errors='replace'), + file_obj) + break + + def IE_sanitize(self, filename): + """cleanup filename from IE full paths""" + return filename and filename[filename.rfind("\\")+1:].strip() + + + +class LazyStream(object): + """ + The LazyStream wrapper allows one to pull and "unget" bytes from a stream. + + Given a producer object (an iterator that yields bytestrings), the + LazyStream object will support iteration, reading, and keeping a + "look-back" variable in case you need to "unget" some bytes. + """ + def __init__(self, producer, length=None): + """ + Every LazyStream must have a producer when instantiated. + + A producer is an iterable that returns a string each time it + is called. + """ + self._producer = producer + self._empty = False + self._leftover = '' + self.length = length + self.position = 0 + self._remaining = length + + def tell(self): + return self.position + + def read(self, size=None): + def parts(): + remaining = (size is not None and [size] or [self._remaining])[0] + # do the whole thing in one shot if no limit was provided. + if remaining is None: + yield ''.join(self) + return + + # otherwise do some bookkeeping to return exactly enough + # of the stream and stashing any extra content we get from + # the producer + while remaining != 0: + assert remaining > 0, 'remaining bytes to read should never go negative' + + chunk = self.next() + + emitting = chunk[:remaining] + self.unget(chunk[remaining:]) + remaining -= len(emitting) + yield emitting + + out = ''.join(parts()) + self.position += len(out) + return out + + def next(self): + """ + Used when the exact number of bytes to read is unimportant. + + This procedure just returns whatever is chunk is conveniently + returned from the iterator instead. Useful to avoid + unnecessary bookkeeping if performance is an issue. + """ + if self._leftover: + output = self._leftover + self.position += len(output) + self._leftover = '' + return output + else: + output = self._producer.next() + self.position += len(output) + return output + + def close(self): + """ + Used to invalidate/disable this lazy stream. + + Replaces the producer with an empty list. Any leftover bytes + that have already been read will still be reported upon read() + and/or next(). + """ + self._producer = [] + + def __iter__(self): + return self + + def unget(self, bytes): + """ + Places bytes back onto the front of the lazy stream. + + Future calls to read() will return those bytes first. The + stream position and thus tell() will be rewound. + """ + self.position -= len(bytes) + self._leftover = ''.join([bytes, self._leftover]) + + def exhaust(self): + """ + Exhausts the entire underlying stream. + + Useful for skipping and advancing sections. + """ + for thing in self: + pass + + +class ChunkIter(object): + """ + An iterable that will yield chunks of data. + Given a file-like object as the constructor, + this object will yield chunks of read operations + from that object. + """ + def __init__(self, flo, chunk_size=64 * 1024): + self.flo = flo + self.chunk_size = chunk_size + + def next(self): + try: + data = self.flo.read(self.chunk_size) + except InputStreamExhausted: + raise StopIteration + if data: + return data + else: + raise StopIteration + + def __iter__(self): + return self + + +class LimitBytes(object): + """ Limit bytes for a file object. """ + def __init__(self, fileobject, length): + self._file = fileobject + self.remaining = length + + def read(self, num_bytes=None): + """ + Read data from the underlying file. + If you ask for too much or there isn't anything left, + this will raise an InputStreamExhausted error. + """ + if self.remaining <= 0: + raise InputStreamExhausted() + if num_bytes is None: + num_bytes = self.remaining + else: + num_bytes = min(num_bytes, self.remaining) + self.remaining -= num_bytes + return self._file.read(num_bytes) + + def exhaust(self): + """ + Exhaust this file until all of the bytes it was limited by + have been read. + """ + while self.remaining > 0: + num_bytes = min(self.remaining, 16384) + __ = self._file.read(num_bytes) + self.remaining -= num_bytes + + +class InterBoundaryIter(object): + """ + A Producer that will iterate over boundaries. + """ + def __init__(self, stream, boundary): + self._stream = stream + self._boundary = boundary + + def __iter__(self): + return self + + def next(self): + try: + return LazyStream(BoundaryIter(self._stream, self._boundary)) + except InputStreamExhausted: + raise StopIteration + + +class BoundaryIter(object): + """ + A Producer that is sensitive to boundaries. + + Will happily yield bytes until a boundary is found. Will yield the + bytes before the boundary, throw away the boundary bytes + themselves, and push the post-boundary bytes back on the stream. + + The future calls to .next() after locating the boundary will raise + a StopIteration exception. + """ + + def __init__(self, stream, boundary): + self._stream = stream + self._boundary = boundary + self._done = False + # rollback an additional six bytes because the format is like + # this: CRLF[--CRLF] + self._rollback = len(boundary) + 6 + + # Try to use mx fast string search if available. Otherwise + # use Python find. Wrap the latter for consistency. + unused_char = self._stream.read(1) + if not unused_char: + raise InputStreamExhausted + self._stream.unget(unused_char) + try: + from mx.TextTools import FS + self._fs = FS(boundary).find + except ImportError: + self._fs = lambda data: data.find(boundary) + + def __iter__(self): + return self + + def next(self): + if self._done: + raise StopIteration + + stream = self._stream + rollback = self._rollback + + bytes_read = 0 + chunks = [] + for bytes in stream: + bytes_read += len(bytes) + chunks.append(bytes) + if bytes_read > rollback: + break + if not bytes: + break + else: + self._done = True + + if not chunks: + raise StopIteration + + chunk = ''.join(chunks) + boundary = self._find_boundary(chunk, len(chunk) < self._rollback) + + if boundary: + end, next = boundary + stream.unget(chunk[next:]) + self._done = True + return chunk[:end] + else: + # make sure we dont treat a partial boundary (and + # its separators) as data + if not chunk[:-rollback]:# and len(chunk) >= (len(self._boundary) + 6): + # There's nothing left, we should just return and mark as done. + self._done = True + return chunk + else: + stream.unget(chunk[-rollback:]) + return chunk[:-rollback] + + def _find_boundary(self, data, eof = False): + """ + Finds a multipart boundary in data. + + Should no boundry exist in the data None is returned + instead. Otherwise a tuple containing + the indices of the following are returned: + + * the end of current encapsulation + + * the start of the next encapsulation + """ + index = self._fs(data) + if index < 0: + return None + else: + end = index + next = index + len(self._boundary) + data_len = len(data) - 1 + # backup over CRLF + if data[max(0,end-1)] == '\n': end -= 1 + if data[max(0,end-1)] == '\r': end -= 1 + # skip over --CRLF + if data[min(data_len,next)] == '-': next += 1 + if data[min(data_len,next)] == '-': next += 1 + if data[min(data_len,next)] == '\r': next += 1 + if data[min(data_len,next)] == '\n': next += 1 + return end, next + + +def ParseBoundaryStream(stream, max_header_size): + """ + Parses one and exactly one stream that encapsulates a boundary. + """ + # Stream at beginning of header, look for end of header + # and parse it if found. The header must fit within one + # chunk. + chunk = stream.read(max_header_size) + # 'find' returns the top of these four bytes, so we'll + # need to munch them later to prevent them from polluting + # the payload. + header_end = chunk.find('\r\n\r\n') + + def parse_header(line): + from cgi import parse_header + main_value_pair, params = parse_header(line) + try: + name, value = main_value_pair.split(':', 1) + except: + raise ValueError("Invalid header: %r" % line) + return name, (value, params) + + if header_end == -1: + # we find no header, so we just mark this fact and pass on + # the stream verbatim + stream.unget(chunk) + return ('RAW', {}, stream) + + header = chunk[:header_end] + + # here we place any excess chunk back onto the stream, as + # well as throwing away the CRLFCRLF bytes from above. + stream.unget(chunk[header_end + 4:]) + + is_file_field = False + outdict = {} + + # Eliminate blank lines + for line in header.split('\r\n'): + # This terminology ("main value" and "dictionary of + # parameters") is from the Python docs. + name, (value, params) = parse_header(line) + if name == 'content-disposition' and params.get('filename'): + is_file_field = True + + outdict[name] = value, params + + if is_file_field: + return ('FILE', outdict, stream) + else: + return ('FIELD', outdict, stream) + + +class Parser(object): + def __init__(self, stream, boundary): + self._stream = stream + self._separator = '--' + boundary + + def __iter__(self): + + boundarystream = InterBoundaryIter(self._stream, + self._separator) + + for sub_stream in boundarystream: + # Iterate over each part + yield ParseBoundaryStream(sub_stream, 1024) Index: django/http/__init__.py =================================================================== --- django/http/__init__.py (revision 7359) +++ django/http/__init__.py (working copy) @@ -11,7 +11,7 @@ from django.utils.datastructures import MultiValueDict, FileDict from django.utils.encoding import smart_str, iri_to_uri, force_unicode - +from django.http.multipartparser import MultiPartParser from utils import * RESERVED_CHARS="!*'();:@&=+$,/?%#[]" @@ -30,6 +30,7 @@ self.GET, self.POST, self.COOKIES, self.META, self.FILES = {}, {}, {}, {}, {} self.path = '' self.method = None + self._upload_handlers = [] def __repr__(self): return '' % \ @@ -102,40 +103,38 @@ encoding = property(_get_encoding, _set_encoding) -def parse_file_upload(header_dict, post_data): - """Returns a tuple of (POST QueryDict, FILES MultiValueDict).""" - import email, email.Message - from cgi import parse_header - raw_message = '\r\n'.join(['%s:%s' % pair for pair in header_dict.items()]) - raw_message += '\r\n\r\n' + post_data - msg = email.message_from_string(raw_message) - POST = QueryDict('', mutable=True) - FILES = MultiValueDict() - for submessage in msg.get_payload(): - if submessage and isinstance(submessage, email.Message.Message): - name_dict = parse_header(submessage['Content-Disposition'])[1] - # name_dict is something like {'name': 'file', 'filename': 'test.txt'} for file uploads - # or {'name': 'blah'} for POST fields - # We assume all uploaded files have a 'filename' set. - if 'filename' in name_dict: - assert type([]) != type(submessage.get_payload()), "Nested MIME messages are not supported" - if not name_dict['filename'].strip(): - continue - # IE submits the full path, so trim everything but the basename. - # (We can't use os.path.basename because that uses the server's - # directory separator, which may not be the same as the - # client's one.) - filename = name_dict['filename'][name_dict['filename'].rfind("\\")+1:] - FILES.appendlist(name_dict['name'], FileDict({ - 'filename': filename, - 'content-type': 'Content-Type' in submessage and submessage['Content-Type'] or None, - 'content': submessage.get_payload(), - })) - else: - POST.appendlist(name_dict['name'], submessage.get_payload()) - return POST, FILES + def _set_upload_handlers(self, upload_handlers): + """ + Set the upload handler to the new handler given in the parameter. + """ + if hasattr(self, '_files'): + raise AttributeError("You cannot set the upload handler after the upload has been processed.") + self._upload_handlers = upload_handlers + def _get_upload_handlers(self): + return self._upload_handlers + upload_handlers = property(_get_upload_handlers, _set_upload_handlers) + + def parse_file_upload(self, META, post_data): + """Returns a tuple of (POST QueryDict, FILES MultiValueDict).""" + from django.core.files.fileuploadhandler import TemporaryFileUploadHandler, MemoryFileUploadHandler + if not self.upload_handlers: + # The order of the upload handlers specifies the order in which + # the handlers receive data. One handler may or may not + # prevent the execution of subsequent handlers. + self.upload_handlers = (MemoryFileUploadHandler(), + TemporaryFileUploadHandler()) + else: + # By using tuple(), we render the upload_handlers + # data structure immutable. + self.upload_handlers = tuple(self.upload_handlers) + + parser = MultiPartParser(META, post_data, self.upload_handlers, + self.encoding) + return parser.parse() + + class QueryDict(MultiValueDict): """ A specialized MultiValueDict that takes a query string when initialized. Index: django/test/client.py =================================================================== --- django/test/client.py (revision 7359) +++ django/test/client.py (working copy) @@ -18,6 +18,25 @@ BOUNDARY = 'BoUnDaRyStRiNg' MULTIPART_CONTENT = 'multipart/form-data; boundary=%s' % BOUNDARY +class FakePayload(object): + """ + A wrapper around StringIO that restricts what can be read, + since data from the network can't be seeked and cannot + be read outside of its content length (or else we hang). + """ + def __init__(self, content): + self.__content = StringIO(content) + self.__len = len(content) + + def read(self, num_bytes=None): + if num_bytes is None: + num_bytes = self.__len or 1 + assert self.__len >= num_bytes, "Cannot read more than the available bytes from the HTTP incoming data." + content = self.__content.read(num_bytes) + self.__len -= num_bytes + return content + + class ClientHandler(BaseHandler): """ A HTTP Handler that can be used for testing purposes. @@ -230,7 +249,7 @@ 'CONTENT_TYPE': content_type, 'PATH_INFO': urllib.unquote(path), 'REQUEST_METHOD': 'POST', - 'wsgi.input': StringIO(post_data), + 'wsgi.input': FakePayload(post_data), } r.update(extra) Index: django/conf/global_settings.py =================================================================== --- django/conf/global_settings.py (revision 7359) +++ django/conf/global_settings.py (working copy) @@ -224,6 +224,11 @@ # Example: "http://media.lawrence.com" MEDIA_URL = '' +# Directory to upload streamed files temporarily. +# A value of `None` means that it will use the default temporary +# directory for the server's operating system. +FILE_UPLOAD_TEMP_DIR = None + # Default formatting for date objects. See all available format strings here: # http://www.djangoproject.com/documentation/templates/#now DATE_FORMAT = 'N j, Y' Index: django/db/models/base.py =================================================================== --- django/db/models/base.py (revision 7359) +++ django/db/models/base.py (working copy) @@ -13,6 +13,7 @@ from django.utils.datastructures import SortedDict from django.utils.functional import curry from django.utils.encoding import smart_str, force_unicode, smart_unicode +from django.core.files.filemove import file_move_safe from django.conf import settings from itertools import izip import types @@ -384,12 +385,16 @@ def _get_FIELD_size(self, field): return os.path.getsize(self._get_FIELD_filename(field)) - def _save_FIELD_file(self, field, filename, raw_contents, save=True): + def _save_FIELD_file(self, field, filename, raw_field, save=True): directory = field.get_directory_name() try: # Create the date-based directory if it doesn't exist. os.makedirs(os.path.join(settings.MEDIA_ROOT, directory)) except OSError: # Directory probably already exists. pass + + if filename is None: + filename = raw_field.file_name + filename = field.get_filename(filename) # If the filename already exists, keep adding an underscore to the name of @@ -406,9 +411,18 @@ setattr(self, field.attname, filename) full_filename = self._get_FIELD_filename(field) - fp = open(full_filename, 'wb') - fp.write(raw_contents) - fp.close() + if hasattr(raw_field, 'temporary_file_path'): + raw_field.close() + file_move_safe(raw_field.temporary_file_path(), full_filename) + else: + from django.core.files import filelocks + fp = open(full_filename, 'wb') + # exclusive lock + filelocks.lock(fp, filelocks.LOCK_EX) + # Stream it into the file, from where it is. + for chunk in raw_field.chunk(65535): + fp.write(chunk) + fp.close() # Save the width and/or height, if applicable. if isinstance(field, ImageField) and (field.width_field or field.height_field): Index: django/db/models/fields/__init__.py =================================================================== --- django/db/models/fields/__init__.py (revision 7359) +++ django/db/models/fields/__init__.py (working copy) @@ -785,7 +785,8 @@ setattr(cls, 'get_%s_filename' % self.name, curry(cls._get_FIELD_filename, field=self)) setattr(cls, 'get_%s_url' % self.name, curry(cls._get_FIELD_url, field=self)) setattr(cls, 'get_%s_size' % self.name, curry(cls._get_FIELD_size, field=self)) - setattr(cls, 'save_%s_file' % self.name, lambda instance, filename, raw_contents, save=True: instance._save_FIELD_file(self, filename, raw_contents, save)) + setattr(cls, 'save_%s_file' % self.name, lambda instance, filename, raw_field, save=True: instance._save_FIELD_file(self, filename, raw_field, save)) + setattr(cls, 'move_%s_file' % self.name, lambda instance, raw_field, save=True: instance._save_FIELD_file(self, None, raw_field, save)) dispatcher.connect(self.delete_file, signal=signals.post_delete, sender=cls) def delete_file(self, instance): @@ -808,9 +809,9 @@ if new_data.get(upload_field_name, False): func = getattr(new_object, 'save_%s_file' % self.name) if rel: - func(new_data[upload_field_name][0]["filename"], new_data[upload_field_name][0]["content"], save) + func(new_data[upload_field_name][0].file_name, new_data[upload_field_name][0], save) else: - func(new_data[upload_field_name]["filename"], new_data[upload_field_name]["content"], save) + func(new_data[upload_field_name].file_name, new_data[upload_field_name], save) def get_directory_name(self): return os.path.normpath(force_unicode(datetime.datetime.now().strftime(smart_str(self.upload_to)))) @@ -823,7 +824,7 @@ def save_form_data(self, instance, data): from django.newforms.fields import UploadedFile if data and isinstance(data, UploadedFile): - getattr(instance, "save_%s_file" % self.name)(data.filename, data.content, save=False) + getattr(instance, "save_%s_file" % self.name)(data.filename, data.data, save=False) def formfield(self, **kwargs): defaults = {'form_class': forms.FileField} Index: django/oldforms/__init__.py =================================================================== --- django/oldforms/__init__.py (revision 7359) +++ django/oldforms/__init__.py (working copy) @@ -680,18 +680,23 @@ self.field_name, self.is_required = field_name, is_required self.validator_list = [self.isNonEmptyFile] + validator_list - def isNonEmptyFile(self, field_data, all_data): - try: - content = field_data['content'] - except TypeError: - raise validators.CriticalValidationError, ugettext("No file was submitted. Check the encoding type on the form.") - if not content: + def isNonEmptyFile(self, new_data, all_data): + if hasattr(new_data, 'upload_errors'): + upload_errors = new_data.upload_errors() + if upload_errors: + raise validators.CriticalValidationError, upload_errors + if not new_data.file_size: raise validators.CriticalValidationError, ugettext("The submitted file is empty.") def render(self, data): return mark_safe(u'' % \ (self.get_id(), self.__class__.__name__, self.field_name)) + def prepare(self, new_data): + if hasattr(new_data, 'upload_errors'): + upload_errors = new_data.upload_errors() + new_data[self.field_name] = { '_file_upload_error': upload_errors } + def html2python(data): if data is None: raise EmptyValue Index: django/core/handlers/wsgi.py =================================================================== --- django/core/handlers/wsgi.py (revision 7359) +++ django/core/handlers/wsgi.py (working copy) @@ -78,6 +78,7 @@ self.path = force_unicode(environ['PATH_INFO']) self.META = environ self.method = environ['REQUEST_METHOD'].upper() + self._upload_handlers = [] def __repr__(self): # Since this is called as part of error handling, we need to be very @@ -112,9 +113,8 @@ # Populates self._post and self._files if self.method == 'POST': if self.environ.get('CONTENT_TYPE', '').startswith('multipart'): - header_dict = dict([(k, v) for k, v in self.environ.items() if k.startswith('HTTP_')]) - header_dict['Content-Type'] = self.environ.get('CONTENT_TYPE', '') - self._post, self._files = http.parse_file_upload(header_dict, self.raw_post_data) + self._raw_post_data = '' + self._post, self._files = self.parse_file_upload(self.META, self.environ['wsgi.input']) else: self._post, self._files = http.QueryDict(self.raw_post_data, encoding=self._encoding), datastructures.MultiValueDict() else: Index: django/core/handlers/modpython.py =================================================================== --- django/core/handlers/modpython.py (revision 7359) +++ django/core/handlers/modpython.py (working copy) @@ -16,6 +16,7 @@ def __init__(self, req): self._req = req self.path = force_unicode(req.uri) + self._upload_handlers = [] def __repr__(self): # Since this is called as part of error handling, we need to be very @@ -53,7 +54,8 @@ def _load_post_and_files(self): "Populates self._post and self._files" if 'content-type' in self._req.headers_in and self._req.headers_in['content-type'].startswith('multipart'): - self._post, self._files = http.parse_file_upload(self._req.headers_in, self.raw_post_data) + self._raw_post_data = '' + self._post, self._files = self.parse_file_upload(self.META, self._req) else: self._post, self._files = http.QueryDict(self.raw_post_data, encoding=self._encoding), datastructures.MultiValueDict() Index: django/core/files/filelocks.py =================================================================== --- django/core/files/filelocks.py (revision 0) +++ django/core/files/filelocks.py (revision 0) @@ -0,0 +1,68 @@ +""" +Locking portability based partially on example by +Jonathan Feignberg in python cookbook. + +Example Usage:: + + from django.utils import file_locks + + f = open('./file', 'wb') + + file_locks.lock(f, file_locks.LOCK_EX) + f.write('Django') + f.close() +""" + +__all__ = ('LOCK_EX','LOCK_SH','LOCK_NB','lock','unlock') + +system_type = None + +try: + import win32con + import win32file + import pywintypes + LOCK_EX = win32con.LOCKFILE_EXCLUSIVE_LOCK + LOCK_SH = 0 + LOCK_NB = win32con.LOCKFILE_FAIL_IMMEDIATELY + __overlapped = pywintypes.OVERLAPPED() + system_type = 'nt' +except (ImportError, AttributeError): + pass + +try: + import fcntl + LOCK_EX = fcntl.LOCK_EX + LOCK_SH = fcntl.LOCK_SH + LOCK_NB = fcntl.LOCK_NB + system_type = 'posix' +except (ImportError, AttributeError): + pass + + + +if system_type == 'nt': + def lock(file, flags): + hfile = win32file._get_osfhandle(file.fileno()) + win32file.LockFileEx(hfile, flags, 0, -0x10000, __overlapped) + + def unlock(file): + hfile = win32file._get_osfhandle(file.fileno()) + win32file.UnlockFileEx(hfile, 0, -0x10000, __overlapped) + +elif system_type =='posix': + def lock(file, flags): + fcntl.flock(file.fileno(), flags) + + def unlock(file): + fcntl.flock(file.fileno(), fcntl.LOCK_UN) + +else: + # File locking is not supported. + LOCK_EX = LOCK_SH = LOCK_NB = None + + # Dummy functions that don't do anything. + def lock(file, flags): + pass + + def unlock(file): + pass Index: django/core/files/uploadedfile.py =================================================================== --- django/core/files/uploadedfile.py (revision 0) +++ django/core/files/uploadedfile.py (revision 0) @@ -0,0 +1,191 @@ +""" +The uploaded file objects for Django. +This contains the base UploadedFile and the TemporaryUploadedFile +derived class. +""" + +__all__ = ('UploadedFile', 'TemporaryUploadedFile', 'InMemoryUploadedFile') + +class UploadedFile(object): + """ + The UploadedFile object behaves somewhat like a file + object and represents some data that the user submitted + and is stored in some form. + """ + DEFAULT_CHUNK_SIZE = 64 * 2**10 + + def __init__(self): + self.file_size = None + self.file_name = None + self.content_type = None + self.charset = None + pass + + def file_size(self): + return self.file_size + + def chunk(self, chunk_size=None): + """ + Read the file to generate chunks of chunk_size bytes. + """ + if not chunk_size: + chunk_size = UploadedFile.DEFAULT_CHUNK_SIZE + + if hasattr(self, 'seek'): + self.seek(0) + # Assume the pointer is at zero... + counter = self.file_size() + + while counter > 0: + yield self.read(chunk_size) + counter -= chunk_size + + + def multiple_chunks(self, chunk_size=None): + """ + Return True if you can expect multiple chunks, False otherwise. + Note: If a particular file representation is in memory, then + override this to return False. + """ + if not chunk_size: + chunk_size = UploadedFile.DEFAULT_CHUNK_SIZE + return self.file_size() < chunk_size + + + def read(self, num_bytes=None): + """ + Read from the file in whatever representation it has. + """ + raise NotImplementedError() + + def open(self): + """ + Open the file, if one needs to. + """ + pass + + + def close(self): + """ + Close the file, if one needs to. + """ + pass + + def __getitem__(self, key): + """ + This maintains backwards compatibility. + """ + import warnings + warnings.warn("The dictionary access of uploaded file objects is deprecated. Use the new object interface instead.", DeprecationWarning) + # Dictionary to translate labels + # for backwards compatbility. + # Should be removed at some point. + backwards_translate = { + 'filename': 'file_name', + 'content-type': 'content_type', + } + + if key == 'content': + return self.read() + else: + return getattr(self, backwards_translate.get(key, key)) + + def __repr__(self): + """ + This representation could be anything and can be overridden. + This is mostly done to make it look somewhat useful. + """ + _dict = { + 'file_name': self.file_name, + 'content_type': self.content_type, + 'content': '', + } + return repr(_dict) + + +class TemporaryUploadedFile(UploadedFile): + """ + Upload a file to a temporary file. + """ + + def __init__(self, file, file_name, content_type, file_size, charset): + self.file = file + self.file_name = file_name + self.path = file.name + self.content_type = content_type + self.file_size = file_size + self.charset = charset + self.file.seek(0) + + def temporary_file_path(self): + """ + Return the full path of this file. + """ + return self.path + + def read(self, *args, **kwargs): + return self.file.read(*args, **kwargs) + + def open(self): + """ + Assume the person meant to seek. + """ + self.seek(0) + + def seek(self, *args, **kwargs): + self.file.seek(*args, **kwargs) + + +class InMemoryUploadedFile(UploadedFile): + """ + Upload a file into memory. + """ + def __init__(self, file, field_name, file_name, content_type, charset): + self.file = file + self.field_name = field_name + self.file_name = file_name + self.content_type = content_type + self.charset = charset + self.file.seek(0) + + def seek(self, *args, **kwargs): + self.file.seek(*args, **kwargs) + + def open(self): + self.seek(0) + + def read(self, *args, **kwargs): + return self.file.read(*args, **kwargs) + + def chunk(self, chunk_size=None): + """ + Return the entirety of the data regardless. + """ + self.file.seek(0) + return self.read() + + def multiple_chunks(self, chunk_size=None): + """ + Since it's in memory, we'll never have multiple chunks. + """ + return False + + +class SimpleUploadedFile(InMemoryUploadedFile): + """ + A simple representation of a file, which + just has content, size, and a name. + """ + def __init__(self, name, content, content_type='text/plain'): + try: + from cStringIO import StringIO + except ImportError: + from StringIO import StringIO + self.file = StringIO(content or '') + self.file_name = name + self.field_name = None + self.file_size = len(content or '') + self.content_type = content_type + self.charset = None + self.file.seek(0) + Index: django/core/files/__init__.py =================================================================== --- django/core/files/__init__.py (revision 0) +++ django/core/files/__init__.py (revision 0) @@ -0,0 +1 @@ + Index: django/core/files/fileuploadhandler.py =================================================================== --- django/core/files/fileuploadhandler.py (revision 0) +++ django/core/files/fileuploadhandler.py (revision 0) @@ -0,0 +1,243 @@ +""" A fileuploadhandler base and default subclass for handling file uploads. +""" +import os +try: + from cStringIO import StringIO +except ImportError: + from StringIO import StringIO + +from django.utils.encoding import force_unicode +from django.utils.datastructures import MultiValueDict + +from django.core.files.uploadedfile import TemporaryUploadedFile, InMemoryUploadedFile + +__all__ = ('UploadFileException','StopUpload', 'SkipFile', + 'FileUploadHandler', 'TemporaryFileUploadHandler', + 'MemoryFileUploadHandler') + + +class UploadFileException(Exception): + """ Any error having to do with Uploading Files. """ + pass + +class StopUpload(UploadFileException): + """ This exception is raised when an upload must abort. """ + pass + +class SkipFile(UploadFileException): + """ This exception is raised when a file needs to be skipped. """ + pass + + +class FileUploadHandler(object): + """ FileUploadHandler will take data and handle file uploads + in a streamed fashion. + """ + chunk_size = 64 * 2 ** 10 #: The default chunk size is 64 KB. + + def __init__(self): + " Initialize some local variables. " + self.file_name = None + self.content_type = None + self.content_length = None + self.charset = None + + def handle_raw_input(self, input_data, META, content_length, boundary, encoding=None): + """ + Handle the raw input from the client. + Parameters: + *input_data* -- An object that supports reading via .read(). + *content_length* -- The (integer) value of the Content-Length header from the client. + *boundary* -- The boundary from the Content-Type header. Be sure to prepend two '--'. + """ + pass + + def new_file(self, field_name, file_name, content_type, content_length, charset=None): + """ + Signal that a new file has been started. + + Warning: Do not trust content_length, if you get it at all. + """ + self.field_name = field_name + self.file_name = file_name + self.content_type = content_type + self.content_length = content_length + self.charset = charset + + def receive_data_chunk(self, raw_data, start, stop): + """ + Receive data from the streamed upload parser. + Start and stop are the positions in the file. + This equality should always be true:: + len(raw_data) = stop - start + """ + raise NotImplementedError() + + def file_complete(self, file_size): + """ + Signal that a file has completed. + File size corresponds to the actual size accumulated + by all the chunks. + + This should return a valid UploadedFile object. + """ + raise NotImplementedError() + + def upload_complete(self): + """ + Signal that the upload is complete. + Do any cleanup that is necessary for this handler. + """ + pass + + + +class TemporaryFileUploadHandler(FileUploadHandler): + """ + Upload the streaming data into a temporary file. + """ + def __init__(self, *args, **kwargs): + """ Import settings for later. """ + super(TemporaryFileUploadHandler, self).__init__(*args, **kwargs) + global settings + from django.conf import settings + + def new_file(self, file_name, *args, **kwargs): + """ + Create the file object to append to as data is coming in. + """ + super(TemporaryFileUploadHandler, self).new_file(file_name, *args, **kwargs) + self.file = TemporaryFile(settings.FILE_UPLOAD_TEMP_DIR) + self.write = self.file.write + + def receive_data_chunk(self, raw_data, start, stop): + """ + Once we get the data, we will save it to our file. + """ + self.write(raw_data) + + def file_complete(self, file_size): + """ + Signal that a file has completed. + File size corresponds to the actual size accumulated + by all the chunks. + + This should return a valid UploadedFile object. + """ + self.file.seek(0) + return TemporaryUploadedFile(self.file, self.file_name, + self.content_type, file_size, + self.charset) + + +class TemporaryFile(object): + """ + A temporary file that tries to delete itself when garbage collected. + """ + def __init__(self, dir): + import tempfile + if not dir: + dir = tempfile.gettempdir() + try: + (fd, name) = tempfile.mkstemp(suffix='.upload', dir=dir) + self.file = os.fdopen(fd, 'w+b') + except (OSError, IOError): + raise OSError, "Could not create temporary file for uploading, have you set settings.FILE_UPLOAD_TEMP_DIR correctly?" + self.name = name + + def __getattr__(self, name): + a = getattr(self.__dict__['file'], name) + if type(a) != type(0): + setattr(self, name, a) + return a + + def __del__(self): + try: + os.unlink(self.name) + except OSError: + pass + + +class MemoryFileUploadHandler(FileUploadHandler): + """ + The MemoryFileUploadHandler will place the data directly into memory. + """ + + def __init__(self): + pass + + + def handle_raw_input(self, input_data, META, content_length, boundary, encoding=None): + """ + Parse the input data in-memory. + """ + if content_length > 2621440: + # If the post is greater than 2.5 MB, do nothing. + return + + from django.http import QueryDict + import email, email.Message + from cgi import parse_header + + ##### + # Get the headers from the META information. + headers = [] + if 'HTTP_CONTENT_TYPE' not in META: + headers.append('Content-Type: %s' % (META.get('CONTENT_TYPE', ''))) + + if 'HTTP_CONTENT_LENGTH' not in META: + headers.append('Content-Length: %s' % (META.get('CONTENT_LENGTH', '0'))) + + for key, value in META.items(): + if key.startswith('HTTP_'): + headers.append('%s: %s' % (key[5:].replace('_','-').title(), value)) + + raw_message = '\r\n'.join(headers) + raw_message += '\r\n\r\n' + input_data.read() + + msg = email.message_from_string(raw_message) + POST = QueryDict('', mutable=True) + FILES = MultiValueDict() + for submessage in msg.get_payload(): + if submessage and isinstance(submessage, email.Message.Message): + name_dict = parse_header(submessage['Content-Disposition'])[1] + field_name = force_unicode(name_dict['name'], encoding, errors='replace') + + if 'filename' in name_dict: + assert not isinstance(submessage.get_payload(), list), "Nested MIME messages are not supported" + if not name_dict['filename'].strip(): + continue + + filename = force_unicode(name_dict['filename'][name_dict['filename'].rfind("\\")+1:], + encoding, errors='replace') + content_type = 'Content-Type' in submessage and submessage['Content-Type'] or None + + file_obj = InMemoryUploadedFile(StringIO(submessage.get_payload()), + field_name, filename, content_type, None) + + FILES.appendlist(field_name, file_obj) + else: + content = force_unicode(submessage.get_payload(), encoding, errors='replace') + POST.appendlist(field_name, content) + + return POST, FILES + + + def new_file(self, field_name, file_name, content_type, content_length, charset): + """ + Do Nothing. + """ + return + + def receive_data_chunk(self, raw_data, start, stop): + """ + Do nothing. + """ + return raw_data + + def file_complete(self, file_size): + """ + Do nothing. + """ + return + Index: django/core/files/filemove.py =================================================================== --- django/core/files/filemove.py (revision 0) +++ django/core/files/filemove.py (revision 0) @@ -0,0 +1,52 @@ +import os + +__all__ = ('file_move_safe',) + +try: + import shutil + file_move = shutil.move +except ImportError: + file_move = os.rename + +def file_move_safe(old_file_name, new_file_name, chunk_size = 1024*64, allow_overwrite=False): + """ + Moves a file from one location to another in the safest way possible. + + First, it tries using shutils.move, which is OS-dependent but doesn't + break with change of filesystems. Then it tries os.rename, which will + break if it encounters a change in filesystems. Lastly, it streams + it manually from one file to another in python. + + Without ``allow_overwrite``, if the destination file exists, the + file will raise an IOError. + """ + + from django.core.files import filelocks + + if old_file_name == new_file_name: + # No file moving takes place. + return + + if not allow_overwrite and os.path.exists(new_file_name): + raise IOError, "Django does not allow overwriting files." + + try: + file_move(old_file_name, new_file_name) + return + except OSError: # moving to another filesystem + pass + + new_file = open(new_file_name, 'wb') + # exclusive lock + filelocks.lock(new_file, filelocks.LOCK_EX) + old_file = open(old_file_name, 'rb') + current_chunk = None + + while current_chunk != '': + current_chunk = old_file.read(chunk_size) + new_file.write(current_chunk) + + new_file.close() + old_file.close() + + os.remove(old_file_name) Index: django/newforms/fields.py =================================================================== --- django/newforms/fields.py (revision 7359) +++ django/newforms/fields.py (working copy) @@ -416,9 +416,9 @@ class UploadedFile(StrAndUnicode): "A wrapper for files uploaded in a FileField" - def __init__(self, filename, content): + def __init__(self, filename, data): self.filename = filename - self.content = content + self.data = data def __unicode__(self): """ @@ -445,12 +445,10 @@ elif not data and initial: return initial try: - f = UploadedFile(data['filename'], data['content']) - except TypeError: + f = UploadedFile(data.file_name, data) + except (TypeError, AttributeError): raise ValidationError(self.error_messages['invalid']) - except KeyError: - raise ValidationError(self.error_messages['missing']) - if not f.content: + if not f.data.file_size: raise ValidationError(self.error_messages['empty']) return f @@ -470,15 +468,26 @@ elif not data and initial: return initial from PIL import Image - from cStringIO import StringIO + + # We need to get the file, it either has a path + # or we have to read it all into memory... + if hasattr(data, 'temporary_file_path'): + file = data.temporary_file_path() + else: + try: + from cStringIO import StringIO + except ImportError: + from StringIO import StringIO + file = StringIO(data.read()) + try: # load() is the only method that can spot a truncated JPEG, # but it cannot be called sanely after verify() - trial_image = Image.open(StringIO(f.content)) + trial_image = Image.open(file) trial_image.load() # verify() is the only method that can spot a corrupt PNG, # but it must be called immediately after the constructor - trial_image = Image.open(StringIO(f.content)) + trial_image = Image.open(file) trial_image.verify() except Exception: # Python Imaging Library doesn't recognize it as an image raise ValidationError(self.error_messages['invalid_image']) Index: tests/modeltests/model_forms/models.py =================================================================== --- tests/modeltests/model_forms/models.py (revision 7359) +++ tests/modeltests/model_forms/models.py (working copy) @@ -75,6 +75,7 @@ __test__ = {'API_TESTS': """ >>> from django import newforms as forms >>> from django.newforms.models import ModelForm +>>> from django.core.files.uploadedfile import SimpleUploadedFile The bare bones, absolutely nothing custom, basic case. @@ -792,7 +793,7 @@ # Upload a file and ensure it all works as expected. ->>> f = TextFileForm(data={'description': u'Assistance'}, files={'file': {'filename': 'test1.txt', 'content': 'hello world'}}) +>>> f = TextFileForm(data={'description': u'Assistance'}, files={'file': SimpleUploadedFile('test1.txt', 'hello world')}) >>> f.is_valid() True >>> type(f.cleaned_data['file']) @@ -819,7 +820,7 @@ # Override the file by uploading a new one. ->>> f = TextFileForm(data={'description': u'Assistance'}, files={'file': {'filename': 'test2.txt', 'content': 'hello world'}}, instance=instance) +>>> f = TextFileForm(data={'description': u'Assistance'}, files={'file': SimpleUploadedFile('test2.txt', 'hello world')}, instance=instance) >>> f.is_valid() True >>> instance = f.save() @@ -838,7 +839,7 @@ >>> instance.file '' ->>> f = TextFileForm(data={'description': u'Assistance'}, files={'file': {'filename': 'test3.txt', 'content': 'hello world'}}, instance=instance) +>>> f = TextFileForm(data={'description': u'Assistance'}, files={'file': SimpleUploadedFile('test3.txt', 'hello world')}, instance=instance) >>> f.is_valid() True >>> instance = f.save() @@ -858,7 +859,7 @@ >>> image_data = open(os.path.join(os.path.dirname(__file__), "test.png")).read() ->>> f = ImageFileForm(data={'description': u'An image'}, files={'image': {'filename': 'test.png', 'content': image_data}}) +>>> f = ImageFileForm(data={'description': u'An image'}, files={'image': SimpleUploadedFile('test.png', image_data)}) >>> f.is_valid() True >>> type(f.cleaned_data['image']) @@ -885,7 +886,7 @@ # Override the file by uploading a new one. ->>> f = ImageFileForm(data={'description': u'Changed it'}, files={'image': {'filename': 'test2.png', 'content': image_data}}, instance=instance) +>>> f = ImageFileForm(data={'description': u'Changed it'}, files={'image': SimpleUploadedFile('test2.png', image_data)}, instance=instance) >>> f.is_valid() True >>> instance = f.save() @@ -904,7 +905,7 @@ >>> instance.image '' ->>> f = ImageFileForm(data={'description': u'And a final one'}, files={'image': {'filename': 'test3.png', 'content': image_data}}, instance=instance) +>>> f = ImageFileForm(data={'description': u'And a final one'}, files={'image': SimpleUploadedFile('test3.png', image_data)}, instance=instance) >>> f.is_valid() True >>> instance = f.save() Index: tests/regressiontests/bug639/tests.py =================================================================== --- tests/regressiontests/bug639/tests.py (revision 7359) +++ tests/regressiontests/bug639/tests.py (working copy) @@ -9,6 +9,7 @@ from regressiontests.bug639.models import Photo from django.http import QueryDict from django.utils.datastructures import MultiValueDict +from django.core.files.uploadedfile import SimpleUploadedFile class Bug639Test(unittest.TestCase): @@ -21,11 +22,7 @@ # Fake a request query dict with the file qd = QueryDict("title=Testing&image=", mutable=True) - qd["image_file"] = { - "filename" : "test.jpg", - "content-type" : "image/jpeg", - "content" : img - } + qd["image_file"] = SimpleUploadedFile('test.jpg', img, 'image/jpeg') manip = Photo.AddManipulator() manip.do_html2python(qd) @@ -39,4 +36,4 @@ Make sure to delete the "uploaded" file to avoid clogging /tmp. """ p = Photo.objects.get() - os.unlink(p.get_image_filename()) \ No newline at end of file + os.unlink(p.get_image_filename()) Index: tests/regressiontests/forms/error_messages.py =================================================================== --- tests/regressiontests/forms/error_messages.py (revision 7359) +++ tests/regressiontests/forms/error_messages.py (working copy) @@ -1,6 +1,7 @@ # -*- coding: utf-8 -*- tests = r""" >>> from django.newforms import * +>>> from django.core.files.uploadedfile import SimpleUploadedFile # CharField ################################################################### @@ -214,11 +215,11 @@ Traceback (most recent call last): ... ValidationError: [u'INVALID'] ->>> f.clean({}) +>>> f.clean(SimpleUploadedFile('name', None)) Traceback (most recent call last): ... -ValidationError: [u'MISSING'] ->>> f.clean({'filename': 'name', 'content':''}) +ValidationError: [u'EMPTY FILE'] +>>> f.clean(SimpleUploadedFile('name', '')) Traceback (most recent call last): ... ValidationError: [u'EMPTY FILE'] Index: tests/regressiontests/forms/fields.py =================================================================== --- tests/regressiontests/forms/fields.py (revision 7359) +++ tests/regressiontests/forms/fields.py (working copy) @@ -2,6 +2,7 @@ tests = r""" >>> from django.newforms import * >>> from django.newforms.widgets import RadioFieldRenderer +>>> from django.core.files.uploadedfile import SimpleUploadedFile >>> import datetime >>> import time >>> import re @@ -773,12 +774,12 @@ >>> f.clean({}) Traceback (most recent call last): ... -ValidationError: [u'No file was submitted.'] +ValidationError: [u'No file was submitted. Check the encoding type on the form.'] >>> f.clean({}, '') Traceback (most recent call last): ... -ValidationError: [u'No file was submitted.'] +ValidationError: [u'No file was submitted. Check the encoding type on the form.'] >>> f.clean({}, 'files/test3.pdf') 'files/test3.pdf' @@ -788,20 +789,20 @@ ... ValidationError: [u'No file was submitted. Check the encoding type on the form.'] ->>> f.clean({'filename': 'name', 'content': None}) +>>> f.clean(SimpleUploadedFile('name', None)) Traceback (most recent call last): ... ValidationError: [u'The submitted file is empty.'] ->>> f.clean({'filename': 'name', 'content': ''}) +>>> f.clean(SimpleUploadedFile('name', '')) Traceback (most recent call last): ... ValidationError: [u'The submitted file is empty.'] ->>> type(f.clean({'filename': 'name', 'content': 'Some File Content'})) +>>> type(f.clean(SimpleUploadedFile('name', 'Some File Content'))) ->>> type(f.clean({'filename': 'name', 'content': 'Some File Content'}, 'files/test4.pdf')) +>>> type(f.clean(SimpleUploadedFile('name', 'Some File Content'), 'files/test4.pdf')) # URLField ################################################################## Index: tests/regressiontests/forms/forms.py =================================================================== --- tests/regressiontests/forms/forms.py (revision 7359) +++ tests/regressiontests/forms/forms.py (working copy) @@ -1,6 +1,7 @@ # -*- coding: utf-8 -*- tests = r""" >>> from django.newforms import * +>>> from django.core.files.uploadedfile import SimpleUploadedFile >>> import datetime >>> import time >>> import re @@ -1465,7 +1466,7 @@ >>> print f File1:
  • This field is required.
->>> f = FileForm(data={}, files={'file1': {'filename': 'name', 'content':''}}, auto_id=False) +>>> f = FileForm(data={}, files={'file1': SimpleUploadedFile('name', '')}, auto_id=False) >>> print f File1:
  • The submitted file is empty.
@@ -1473,7 +1474,7 @@ >>> print f File1:
  • No file was submitted. Check the encoding type on the form.
->>> f = FileForm(data={}, files={'file1': {'filename': 'name', 'content':'some content'}}, auto_id=False) +>>> f = FileForm(data={}, files={'file1': SimpleUploadedFile('name', 'some content')}, auto_id=False) >>> print f File1: >>> f.is_valid() Index: tests/regressiontests/test_client_regress/views.py =================================================================== --- tests/regressiontests/test_client_regress/views.py (revision 7359) +++ tests/regressiontests/test_client_regress/views.py (working copy) @@ -1,5 +1,6 @@ from django.contrib.auth.decorators import login_required from django.http import HttpResponse, HttpResponseRedirect, HttpResponseServerError +import sha def no_template_view(request): "A simple view that expects a GET request, and returns a rendered template" @@ -10,13 +11,36 @@ Check that a file upload can be updated into the POST dictionary without going pear-shaped. """ + from django.core.files.uploadedfile import UploadedFile form_data = request.POST.copy() form_data.update(request.FILES) - if isinstance(form_data['file_field'], dict) and isinstance(form_data['name'], unicode): + if isinstance(form_data['file_field'], UploadedFile) and isinstance(form_data['name'], unicode): return HttpResponse('') else: return HttpResponseServerError() +def file_upload_view_verify(request): + """ + Use the sha digest hash to verify the uploaded contents. + """ + from django.core.files.uploadedfile import UploadedFile + form_data = request.POST.copy() + form_data.update(request.FILES) + for key, value in form_data.items(): + if key.endswith('_hash'): + continue + if key + '_hash' not in form_data: + continue + submitted_hash = form_data[key + '_hash'] + if isinstance(value, UploadedFile): + new_hash = sha.new(value.read()).hexdigest() + else: + new_hash = sha.new(value).hexdigest() + if new_hash != submitted_hash: + return HttpResponseServerError() + + return HttpResponse('') + def get_view(request): "A simple login protected view" return HttpResponse("Hello world") @@ -37,4 +61,4 @@ def login_protected_redirect_view(request): "A view that redirects all requests to the GET view" return HttpResponseRedirect('/test_client_regress/get_view/') -login_protected_redirect_view = login_required(login_protected_redirect_view) \ No newline at end of file +login_protected_redirect_view = login_required(login_protected_redirect_view) Index: tests/regressiontests/test_client_regress/models.py =================================================================== --- tests/regressiontests/test_client_regress/models.py (revision 7359) +++ tests/regressiontests/test_client_regress/models.py (working copy) @@ -5,6 +5,7 @@ from django.test import Client, TestCase from django.core.urlresolvers import reverse import os +import sha class AssertContainsTests(TestCase): def test_contains(self): @@ -243,6 +244,44 @@ response = self.client.post('/test_client_regress/file_upload/', post_data) self.assertEqual(response.status_code, 200) + def test_large_upload(self): + import tempfile + dir = tempfile.gettempdir() + + (fd, name1) = tempfile.mkstemp(suffix='.file1', dir=dir) + file1 = os.fdopen(fd, 'w+b') + file1.write('a' * (2 ** 21)) + file1.seek(0) + + (fd, name2) = tempfile.mkstemp(suffix='.file2', dir=dir) + file2 = os.fdopen(fd, 'w+b') + file2.write('a' * (10 * 2 ** 20)) + file2.seek(0) + + post_data = { + 'name': 'Ringo', + 'file_field1': file1, + 'file_field2': file2, + } + + for key in post_data.keys(): + try: + post_data[key + '_hash'] = sha.new(post_data[key].read()).hexdigest() + post_data[key].seek(0) + except AttributeError: + post_data[key + '_hash'] = sha.new(post_data[key]).hexdigest() + + response = self.client.post('/test_client_regress/file_upload_verify/', post_data) + + for name in (name1, name2): + try: + os.unlink(name) + except: + pass + + self.assertEqual(response.status_code, 200) + + class LoginTests(TestCase): fixtures = ['testdata'] Index: tests/regressiontests/test_client_regress/urls.py =================================================================== --- tests/regressiontests/test_client_regress/urls.py (revision 7359) +++ tests/regressiontests/test_client_regress/urls.py (working copy) @@ -4,6 +4,7 @@ urlpatterns = patterns('', (r'^no_template_view/$', views.no_template_view), (r'^file_upload/$', views.file_upload_view), + (r'^file_upload_verify/$', views.file_upload_view_verify), (r'^get_view/$', views.get_view), url(r'^arg_view/(?P.+)/$', views.view_with_argument, name='arg_view'), (r'^login_protected_redirect_view/$', views.login_protected_redirect_view) Index: AUTHORS =================================================================== --- AUTHORS (revision 7359) +++ AUTHORS (working copy) @@ -58,7 +58,7 @@ Jökull Sólberg Auðunsson Arthur David Avsajanishvili - axiak@mit.edu + Mike Axiak Niran Babalola Morten Bagai Mikaël Barbero @@ -135,6 +135,7 @@ Marc Fargas Szilveszter Farkas favo@exoweb.net + fdr Dmitri Fedortchenko Bill Fenner Stefane Fermgier Index: docs/upload_handling.txt =================================================================== --- docs/upload_handling.txt (revision 0) +++ docs/upload_handling.txt (revision 0) @@ -0,0 +1,148 @@ +=============== +Upload Handlers +=============== + +Upload handlers are components to modify how Django handles file uploads. Through the upload handling framework, Django allows complete control over what happens during and after uploads. + +An upload handler can do many different things with the uploaded data. For instance, one can stream your uploaded data to disk, while another can generate the data to render a Progress Bar for your uploads. + +There are two pieces to the Django upload handling: the upload handler and the uploaded file. These are both represented by python classes -- ``FileUploadHandler`` and ``UploadedFile`` respectively. Throughout the lifetime of the upload process, Django will call on the upload handler to handle the upload, while the upload handler is expected to return an ``UploadedFile`` object at the end of a file's upload. + +Adding Upload Handlers to your Request +====================================== + +To use an upload handler, you have to register it in your request. The request object contains a list of handlers (``request.upload_handlers``) that will be called in order. To append a handler to the list, you would append to it like any other list. For example, suppose you had an ``ProgressBarUploadHandler`` class. To append it to your upload handlers you would write:: + + request.upload_handlers.append(ProgressBarUploadHandler()) + +However, since the progress bar handler would probably need to run before the other handlers get a chance, you'd probably want to insert it. That is, you'd write:: + + request.upload_handlers.insert(0, ProgressBarUploadHandler()) + +If you want to replace the upload handlers completely, you can just assign a new list:: + + request.upload_handlers = [ProgressBarUploadHandler()] + +And all Django will do is keep a progress, but do nothing with the file itself! One more note: After the upload has completed, you are no longer allowed to assign or modify this list, and Django will raise an error if you try to modify this list after an upload. + +Writing a File Upload Handler +============================= + +All file upload handlers are subclasses of ``FileUploadHandler``, found in ``django.core.files.fileuploadhandler``. To create your handler, you need to define the required methods followed by the methods that make most sense to you: + +chunk_size +---------- + +This is an integer attribute that specifies how large the chunks we should put into memory from the network are. The chunk sizes should be divisible by ``4`` and should not exceed ``2 ** 31 - 1`` in size. When there are multiple chunk sizes provided by multiple handlers, Django will use the smallest chunk size. + +new_file +-------- + +This signals that a new file is starting. You can initialize a file or whatever else is needed on a new file upload. + +Interface: ``new_file(self, field_name, file_name, content_type, content_length, charset)`` + +``field_name`` is a string name of the field this was POSTed as. + +``file_name`` is the unicode filename that was provided by the browser. + +``content_type`` is the MIME type provided by the browser -- E.g. ``'image/jpeg'``. + +``content_length`` is the length of the image given by the browser if provided, ``None`` otherwise. + +``charset`` is the charset given by the browser if provided, ``None`` otherwise. + +Returns: ``None`` if you want other handlers to get ``new_file`` called. Something nonzero if you don't want subsequent handlers to get a chance. + +receive_data_chunk +------------------ +*required* + +This method is used to do something with the new chunk of data. For example: The ``TemporaryFileUploadHandler`` takes the data and writes it to disk. + +Interface: ``receive_data_chunk(self, raw_data, start, stop)`` + +``raw_data`` is a byte string containing the uploaded data. + +``start`` is the byte number where the chunk starts. + +``stop`` is the byte number where the chunk stops. + +Returns: ``None`` if you don't want the subsequent upload handlers to receive the data. Whatever else you return gets fed into the subsequent upload handlers' ``receive_data_chunk`` method. + +Exceptions: If you raise a ``StopUpload`` or a ``SkipFile`` exception, the upload will abort or the file will be skipped respectively. + +file_complete +------------- +*required* + +This method defines when a function has finished uploading and gets packaged into an ``UploadedFile`` object. + +Interface: ``file_complete(self, file_size)`` + +``file_size`` is the number of bytes you have received for this file. + +Returns: An ``UploadedFile`` object to set in the ``request.FILES`` dictionary. ``None`` if you want the subsequent upload handlers to get an ``UploadedFile`` object. + +upload_complete +--------------- + +This method defines when the entire upload has completed. + +Interface: ``upload_complete(self)`` + +handle_raw_input +---------------- + +This method allows the handler to completely override the parsing of the raw HTTP-layered input. + +Interface: ``handle_raw_input(self, input_data, META, content_length, boundary, encoding)`` + +``input_data`` is a file-like object that supports the read operation. + +``META`` is the same object as ``request.META``. + +``content_length`` is the length of the data in ``input_data``. Don't read more than ``content_length`` bytes from ``input_data``. + +``boundary`` is the MIME boundary for this request. + +``encoding`` is the encoding of the request. + +Returns: ``None`` if you want to go on to the next stage, ``(POST, FILES)`` if you want to return the new data structures suitable for the request directly. + +Defining an Uploaded File +========================= + +All file upload handlers are subclasses of ``UploadedFile``, found in ``django.core.files.uploadedfile``. The uploaded file object is returned by the handler above in ``file_complete``. To create your own uploaded file class, you need to define the required methods followed by the methods that make most sense to you: + +read +---- +*required* + +Interface: ``read(self, num_bytes=None)`` + +Returns: A byte string of length ``num_bytes`` (or the size of the file if ``num_bytes`` is not supplied). + +chunk +----- + +A generator to yield small chunks from the file. With the ``read()`` defined, the ``UploadedFile`` class already defines a ``chunk()`` method that's probably suitable. + +Interface: ``chunk(self, chunk_size=None)`` + + +multiple_chunks +--------------- + +Interface: ``multiple_chunks(self, chunk_size=None)`` + +Returns: ``True`` or ``False`` depending on whether or not the user of this file can expect more than one chunk when calling ``chunk(self, chunk_size)``. If all the data is in memory, you should return ``False``. + +temporary_file_path +------------------- + +If defined, this method should return the file path of the file on the operating system. This will let users move files rather than write to new files if the option is available. + +Interface: ``temporary_file_path(self)`` + +Returns: A file path in a file system on the local computer. Index: docs/settings.txt =================================================================== --- docs/settings.txt (revision 7359) +++ docs/settings.txt (working copy) @@ -513,6 +513,15 @@ The character encoding used to decode any files read from disk. This includes template files and initial SQL data files. +FILE_UPLOAD_TEMP_DIR +-------------------- + +**New in Django development version** + +Default: ``None`` + +The directory to store data temporarily while uploading files. If ``None``, Django will use the standard temporary directory for the operating system. For example, this will default to '/tmp' on *nix-style operating systems. + FIXTURE_DIRS -------------