Index: django/http/__init__.py =================================================================== --- django/http/__init__.py (revision 371) +++ django/http/__init__.py (working copy) @@ -1,4 +1,5 @@ import os +import cgi from Cookie import SimpleCookie from pprint import pformat from urllib import urlencode, quote @@ -42,12 +43,12 @@ def is_secure(self): return os.environ.get("HTTPS") == "on" -def parse_file_upload(header_dict, post_data): +def default_parse_file_upload(req): "Returns a tuple of (POST MultiValueDict, FILES MultiValueDict)" import email, email.Message from cgi import parse_header - raw_message = '\r\n'.join(['%s:%s' % pair for pair in header_dict.items()]) - raw_message += '\r\n\r\n' + post_data + raw_message = '\r\n'.join(['%s:%s' % pair for pair in req.header_dict.items()]) + raw_message += '\r\n\r\n' + req.raw_post_data msg = email.message_from_string(raw_message) POST = MultiValueDict() FILES = MultiValueDict() @@ -73,6 +74,14 @@ POST.appendlist(name_dict['name'], submessage.get_payload()) return POST, FILES +def parse_file_upload(req): + "Returns a tuple of (POST MultiValueDict, FILES MultiValueDict)" + + if hasattr(req, 'parse_file_upload'): + return req.parse_file_upload(req) + + return default_parse_file_upload(req) + class QueryDict(MultiValueDict): """A specialized MultiValueDict that takes a query string when initialized. This is immutable unless you create a copy of it.""" Index: django/oldforms/__init__.py =================================================================== --- django/oldforms/__init__.py (revision 371) +++ django/oldforms/__init__.py (working copy) @@ -661,10 +661,12 @@ self.validator_list = [self.isNonEmptyFile] + validator_list def isNonEmptyFile(self, field_data, all_data): + if not isinstance(field_data, dict): + raise validators.CriticalValidationError, gettext("No file was submitted. Check the encoding type on the form.") try: + content = field_data.tell()>0 + except: content = field_data['content'] - except TypeError: - raise validators.CriticalValidationError, gettext("No file was submitted. Check the encoding type on the form.") if not content: raise validators.CriticalValidationError, gettext("The submitted file is empty.") Index: django/db/models/base.py =================================================================== --- django/db/models/base.py (revision 371) +++ django/db/models/base.py (working copy) @@ -322,7 +322,7 @@ def _get_FIELD_size(self, field): return os.path.getsize(self._get_FIELD_filename(field)) - def _save_FIELD_file(self, field, filename, raw_contents): + def _save_FIELD_file(self, field, filename, raw_field): directory = field.get_directory_name() try: # Create the date-based directory if it doesn't exist. os.makedirs(os.path.join(settings.MEDIA_ROOT, directory)) @@ -344,9 +344,12 @@ setattr(self, field.attname, filename) full_filename = self._get_FIELD_filename(field) - fp = open(full_filename, 'wb') - fp.write(raw_contents) - fp.close() + if not raw_field.get('payload', False): + fp = open(full_filename, 'wb') + fp.write(raw_field['content']) + fp.close() + else: + os.rename(raw_field['payload']['name'], full_filename) # Save the width and/or height, if applicable. if isinstance(field, ImageField) and (field.width_field or field.height_field): Index: django/db/models/fields/__init__.py =================================================================== --- django/db/models/fields/__init__.py (revision 371) +++ django/db/models/fields/__init__.py (working copy) @@ -648,9 +648,9 @@ if new_data.get(upload_field_name, False): func = getattr(new_object, 'save_%s_file' % self.name) if rel: - func(new_data[upload_field_name][0]["filename"], new_data[upload_field_name][0]["content"]) + func(new_data[upload_field_name][0]["filename"], new_data[upload_field_name][0]) else: - func(new_data[upload_field_name]["filename"], new_data[upload_field_name]["content"]) + func(new_data[upload_field_name]["filename"], new_data[upload_field_name]) def get_directory_name(self): return os.path.normpath(datetime.datetime.now().strftime(self.upload_to)) Index: django/core/handlers/wsgi.py =================================================================== --- django/core/handlers/wsgi.py (revision 371) +++ django/core/handlers/wsgi.py (working copy) @@ -109,9 +109,7 @@ # Populates self._post and self._files if self.method == 'POST': if self.environ.get('CONTENT_TYPE', '').startswith('multipart'): - header_dict = dict([(k, v) for k, v in self.environ.items() if k.startswith('HTTP_')]) - header_dict['Content-Type'] = self.environ.get('CONTENT_TYPE', '') - self._post, self._files = http.parse_file_upload(header_dict, self.raw_post_data) + self._post, self._files = http.parse_file_upload(self) else: self._post, self._files = http.QueryDict(self.raw_post_data), datastructures.MultiValueDict() else: @@ -166,13 +164,23 @@ self._raw_post_data = buf.getvalue() buf.close() return self._raw_post_data + + def _get_raw_request(self): + return self.environ['wsgi.input'] + def _get_header_dict(self): + header_dict = dict([(k, v) for k, v in self.environ.items() if k.startswith('HTTP_')]) + header_dict['Content-Type'] = self.environ.get('CONTENT_TYPE', '') + return header_dict + GET = property(_get_get, _set_get) POST = property(_get_post, _set_post) COOKIES = property(_get_cookies, _set_cookies) FILES = property(_get_files) REQUEST = property(_get_request) raw_post_data = property(_get_raw_post_data) + raw_request = property(_get_raw_request) + header_dict = property(_get_header_dict) class WSGIHandler(BaseHandler): def __call__(self, environ, start_response): Index: django/core/handlers/modpython.py =================================================================== --- django/core/handlers/modpython.py (revision 371) +++ django/core/handlers/modpython.py (working copy) @@ -47,7 +47,7 @@ def _load_post_and_files(self): "Populates self._post and self._files" if self._req.headers_in.has_key('content-type') and self._req.headers_in['content-type'].startswith('multipart'): - self._post, self._files = http.parse_file_upload(self._req.headers_in, self.raw_post_data) + self._post, self._files = http.parse_file_upload(self) else: self._post, self._files = http.QueryDict(self.raw_post_data), datastructures.MultiValueDict() @@ -122,6 +122,12 @@ def _get_method(self): return self.META['REQUEST_METHOD'].upper() + def _get_raw_request(self): + return self._req + + def _get_header_dict(self): + return self._req.headers_in + GET = property(_get_get, _set_get) POST = property(_get_post, _set_post) COOKIES = property(_get_cookies, _set_cookies) @@ -130,6 +136,8 @@ REQUEST = property(_get_request) raw_post_data = property(_get_raw_post_data) method = property(_get_method) + raw_request = property(_get_raw_request) + header_dict = property(_get_header_dict) class ModPythonHandler(BaseHandler): def __call__(self, req): Index: django/contrib/admin/media/js/UploadProgress.js =================================================================== --- django/contrib/admin/media/js/UploadProgress.js (revision 0) +++ django/contrib/admin/media/js/UploadProgress.js (revision 372) @@ -0,0 +1,103 @@ + +function getxy(){ + var x,y; + if (self.innerHeight) // all except Explorer + { + x = self.innerWidth; + y = self.innerHeight; + } + else if (document.documentElement && document.documentElement.clientHeight) + // Explorer 6 Strict Mode + { + x = document.documentElement.clientWidth; + y = document.documentElement.clientHeight; + } + else if (document.body) // other Explorers + { + x = document.body.clientWidth; + y = document.body.clientHeight; + } + return {'x':x,'y':y} + } + +var humanvalue = ['B','KB','MB','GB'] +function humanize(bytes) { + curbytes = bytes + iterations = 0 + while (curbytes>1024) { + iterations++ + curbytes=curbytes/1024 + } + return curbytes.toFixed(1) + ' ' + humanvalue[iterations] + } + +interval = null; +function fetch(uuid) { + req = xmlhttp + req.open("GET", "/progress/", 1); + req.setRequestHeader("X-Progress-Id", uuid); + req.onreadystatechange = function () { + if (req.readyState == 4) { + if (req.status == 200) { + + var upload = eval( '(' + req.responseText + ')' ); + + if (upload.state == 'done' || upload.state == 'uploading') { + bar = document.getElementById('progress_bar'); + bar_txt = document.getElementById('progress_text') + bar_txt.innerHTML = ((upload.received / upload.size) * 100).toFixed(1) + '% - ' + + humanize(upload.received) + ' of ' + humanize(upload.size) + w = 400 * upload.received / upload.size; + bar.style.width = w + 'px'; + + } + if (upload.state == 'done') { + window.clearTimeout(interval); + } + } + } + } + req.send(null); + + } + +function openprogress(e) { + + uuid = ""; + for (i = 0; i < 32; i++) { + uuid += Math.floor(Math.random() * 16).toString(16); + } + frm = e.target||e.srcElement + + frm.action=frm.action+"?" + uuid; + + pos = getxy() + posx = parseInt((pos.x/2)-(420/2), 10) + posy = parseInt((pos.y/2)-(50/2), 10) + + progress_wrap = quickElement('div', document.body, '', 'style', + 'position: absolute; top: '+posy+'px; left: '+posx+'px; height: 50px; ' + + 'padding: 10px; width: 420px; background: #ffffff; ' + + 'border: solid 1px #dddddd;', 'id', 'progress_wrap') + + progress_label = quickElement('h1', progress_wrap, 'Upload progress') + + progress = quickElement('div', progress_wrap, '', 'style', + 'top: 0; left: 0; width: 0px; ', 'id', 'progress_bar', 'class', 'submit-row') + + progress_text = quickElement('div', progress_wrap, '0%', 'style', + 'color: #000000; ', 'id', 'progress_text') + + interval = window.setInterval( + function () { + fetch(uuid); + }, + 1000 + ); + } + +addEvent(window, 'load', function() { + frm = document.getElementsByTagName('form')[0] + addEvent(frm, 'submit', openprogress) + } + ) Index: django/middleware/upload.py =================================================================== --- django/middleware/upload.py (revision 0) +++ django/middleware/upload.py (revision 378) @@ -0,0 +1,417 @@ +"streaming upload middleware" +import cgi +import os +import tempfile +import re +import email, email.Message, email.FeedParser +from email import Errors +from email import Message +from email.FeedParser import NLCRE, NLCRE_bol, NLCRE_eol, NLCRE_crack,headerRE, EMPTYSTRING, NL, NeedMoreData + +from django.conf import settings +from django.utils.datastructures import MultiValueDict +from django.utils import simplejson + +from django.core import signals +from django.dispatch import dispatcher + +def delete_tempfile(sender): + sender.delete() + +class TempFileDict(dict): + "Keeps uploaded file as a file-like object and reads its content on demand" + + def __init__(self): + self['name'] = tempfile.mktemp() + self['file'] = open(self['name'], "w+b") + dispatcher.connect(delete_tempfile, sender=self, signal=signals.request_finished) + + def write(self, data): + self['file'].write(data) + + def close(self): + self['file'].close() + + def delete(self): + if os.path.exsists(self['name']): + os.remove(self['name']) + + def tell(self): + return os.path.getsize(self['name']) + + def __repr__(self): + return '' + +class FileDict(dict): + + def __init__(self, filename, contenttype, payload): + self['filename'] , self['content-type'], self['payload'] = filename, contenttype, payload + + def __getitem__(self, name): + if name=='content' and not 'content' in self: + size = self.tell() + f = open(self['payload']['name'], 'r') + self['content'] = f.read(size) + f.close() + return dict.__getitem__(self, name) + + def tell(self): + return self['payload'].tell() + + def __repr__(self): + return '' + + +class StreamingFileFeedParser(email.FeedParser.FeedParser): + + def _parsegen(self): + # Create a new message and start by parsing headers. + self._new_message() + headers = [] + # Collect the headers, searching for a line that doesn't match the RFC + # 2822 header or continuation pattern (including an empty line). + for line in self._input: + if line is NeedMoreData: + yield NeedMoreData + continue + if not headerRE.match(line): + # If we saw the RFC defined header/body separator + # (i.e. newline), just throw it away. Otherwise the line is + # part of the body so push it back. + if not NLCRE.match(line): + self._input.unreadline(line) + break + headers.append(line) + # Done with the headers, so parse them and figure out what we're + # supposed to see in the body of the message. + self._parse_headers(headers) + # Headers-only parsing is a backwards compatibility hack, which was + # necessary in the older parser, which could throw errors. All + # remaining lines in the input are thrown into the message body. + if self._headersonly: + lines = [] + while True: + line = self._input.readline() + if line is NeedMoreData: + yield NeedMoreData + continue + if line == '': + break + lines.append(line) + self._cur.set_payload(EMPTYSTRING.join(lines)) + return + if self._cur.get_content_type() == 'message/delivery-status': + # message/delivery-status contains blocks of headers separated by + # a blank line. We'll represent each header block as a separate + # nested message object, but the processing is a bit different + # than standard message/* types because there is no body for the + # nested messages. A blank line separates the subparts. + while True: + self._input.push_eof_matcher(NLCRE.match) + for retval in self._parsegen(): + if retval is NeedMoreData: + yield NeedMoreData + continue + break + msg = self._pop_message() + # We need to pop the EOF matcher in order to tell if we're at + # the end of the current file, not the end of the last block + # of message headers. + self._input.pop_eof_matcher() + # The input stream must be sitting at the newline or at the + # EOF. We want to see if we're at the end of this subpart, so + # first consume the blank line, then test the next line to see + # if we're at this subpart's EOF. + while True: + line = self._input.readline() + if line is NeedMoreData: + yield NeedMoreData + continue + break + while True: + line = self._input.readline() + if line is NeedMoreData: + yield NeedMoreData + continue + break + if line == '': + break + # Not at EOF so this is a line we're going to need. + self._input.unreadline(line) + return + if self._cur.get_content_maintype() == 'message': + # The message claims to be a message/* type, then what follows is + # another RFC 2822 message. + for retval in self._parsegen(): + if retval is NeedMoreData: + yield NeedMoreData + continue + break + self._pop_message() + return + if self._cur.get_content_maintype() == 'multipart': + boundary = self._cur.get_boundary() + if boundary is None: + # The message /claims/ to be a multipart but it has not + # defined a boundary. That's a problem which we'll handle by + # reading everything until the EOF and marking the message as + # defective. + self._cur.defects.append(Errors.NoBoundaryInMultipartDefect()) + lines = [] + for line in self._input: + if line is NeedMoreData: + yield NeedMoreData + continue + lines.append(line) + self._cur.set_payload(EMPTYSTRING.join(lines)) + return + # Create a line match predicate which matches the inter-part + # boundary as well as the end-of-multipart boundary. Don't push + # this onto the input stream until we've scanned past the + # preamble. + separator = '--' + boundary + boundaryre = re.compile( + '(?P' + re.escape(separator) + + r')(?P--)?(?P[ \t]*)(?P\r\n|\r|\n)?$') + capturing_preamble = True + preamble = [] + linesep = False + while True: + line = self._input.readline() + if line is NeedMoreData: + yield NeedMoreData + continue + if line == '': + break + mo = boundaryre.match(line) + if mo: + # If we're looking at the end boundary, we're done with + # this multipart. If there was a newline at the end of + # the closing boundary, then we need to initialize the + # epilogue with the empty string (see below). + if mo.group('end'): + linesep = mo.group('linesep') + break + # We saw an inter-part boundary. Were we in the preamble? + if capturing_preamble: + if preamble: + # According to RFC 2046, the last newline belongs + # to the boundary. + lastline = preamble[-1] + eolmo = NLCRE_eol.search(lastline) + if eolmo: + preamble[-1] = lastline[:-len(eolmo.group(0))] + self._cur.preamble = EMPTYSTRING.join(preamble) + capturing_preamble = False + self._input.unreadline(line) + continue + # We saw a boundary separating two parts. Consume any + # multiple boundary lines that may be following. Our + # interpretation of RFC 2046 BNF grammar does not produce + # body parts within such double boundaries. + while True: + line = self._input.readline() + if line is NeedMoreData: + yield NeedMoreData + continue + mo = boundaryre.match(line) + if not mo: + self._input.unreadline(line) + break + # Recurse to parse this subpart; the input stream points + # at the subpart's first line. + self._input.push_eof_matcher(boundaryre.match) + for retval in self._parsegen(): + if retval is NeedMoreData: + yield NeedMoreData + continue + break + # Because of RFC 2046, the newline preceding the boundary + # separator actually belongs to the boundary, not the + # previous subpart's payload (or epilogue if the previous + # part is a multipart). + if self._last.get_content_maintype() == 'multipart': + epilogue = self._last.epilogue + if epilogue == '': + self._last.epilogue = None + elif epilogue is not None: + mo = NLCRE_eol.search(epilogue) + if mo: + end = len(mo.group(0)) + self._last.epilogue = epilogue[:-end] + else: + payload = self._last.get_payload() + if isinstance(payload, basestring): + mo = NLCRE_eol.search(payload) + if mo: + payload = payload[:-len(mo.group(0))] + self._last.set_payload(payload) + self._input.pop_eof_matcher() + self._pop_message() + # Set the multipart up for newline cleansing, which will + # happen if we're in a nested multipart. + self._last = self._cur + else: + # I think we must be in the preamble + assert capturing_preamble + preamble.append(line) + # We've seen either the EOF or the end boundary. If we're still + # capturing the preamble, we never saw the start boundary. Note + # that as a defect and store the captured text as the payload. + # Everything from here to the EOF is epilogue. + if capturing_preamble: + self._cur.defects.append(Errors.StartBoundaryNotFoundDefect()) + self._cur.set_payload(EMPTYSTRING.join(preamble)) + epilogue = [] + for line in self._input: + if line is NeedMoreData: + yield NeedMoreData + continue + self._cur.epilogue = EMPTYSTRING.join(epilogue) + return + # If the end boundary ended in a newline, we'll need to make sure + # the epilogue isn't None + if linesep: + epilogue = [''] + else: + epilogue = [] + for line in self._input: + if line is NeedMoreData: + yield NeedMoreData + continue + epilogue.append(line) + # Any CRLF at the front of the epilogue is not technically part of + # the epilogue. Also, watch out for an empty string epilogue, + # which means a single newline. + if epilogue: + firstline = epilogue[0] + bolmo = NLCRE_bol.match(firstline) + if bolmo: + epilogue[0] = firstline[len(bolmo.group(0)):] + self._cur.epilogue = EMPTYSTRING.join(epilogue) + return + # Otherwise, it's some non-multipart type, so the entire rest of the + # file contents becomes the payload. + name_dict = cgi.parse_header(self._cur['Content-Disposition'])[1] + if name_dict.has_key('filename'): + tmpfile = TempFileDict() + for line in self._input: + if line is NeedMoreData: + yield NeedMoreData + continue + tmpfile.write(line) + tmpfile.close() + self._cur.set_payload(tmpfile) + else: + lines = [] + for line in self._input: + if line is NeedMoreData: + yield NeedMoreData + continue + lines.append(line) + self._cur.set_payload(EMPTYSTRING.join(lines)) + +def parse_streaming_file_upload(req): + "Returns a tuple of (POST MultiValueDict, FILES MultiValueDict)" + + try: + BUFFER_SIZE=settings.UPLOAD_BUFFER_SIZE + except: + BUFFER_SIZE=200000 + + if hasattr(req, 'upload_state'): + upload_state = req.upload_state(req) + else: + upload_state = None + + raw_headers = '\r\n'.join(['%s:%s' % pair for pair in req.header_dict.items()]) + raw_headers += '\r\n\r\n' + POST = MultiValueDict() + FILES = MultiValueDict() + parser = StreamingFileFeedParser() + parser.feed(raw_headers) + while 1: + # make this a non-blocing read + line=req.raw_request.read(BUFFER_SIZE) + if upload_state: + upload_state.addlen(len(line)) + parser.feed(line) + if line == '': + break + msg=parser.close() + POST = MultiValueDict() + FILES = MultiValueDict() + for submessage in msg.get_payload(): + if isinstance(submessage, email.Message.Message): + name_dict = cgi.parse_header(submessage['Content-Disposition'])[1] + # name_dict is something like {'name': 'file', 'filename': 'test.txt'} for file uploads + # or {'name': 'blah'} for POST fields + # We assume all uploaded files have a 'filename' set. + if name_dict.has_key('filename'): + assert type([]) != type(submessage.get_payload()), "Nested MIME messages are not supported" + if not name_dict['filename'].strip(): + continue + # IE submits the full path, so trim everything but the basename. + # (We can't use os.path.basename because it expects Linux paths.) + filename = name_dict['filename'][name_dict['filename'].rfind("\\")+1:] + FILES.appendlist(name_dict['name'], FileDict( + filename, + (submessage.has_key('Content-Type') and submessage['Content-Type'] or None), + submessage.get_payload() + )) + else: + POST.appendlist(name_dict['name'], submessage.get_payload()) + return POST, FILES + +class StreamingUploadMiddleware: + + def process_request(self, request): + request.parse_file_upload = parse_streaming_file_upload + +def get_temp_file(identifier): + return os.path.join(tempfile.gettempdir(),identifier) + +class UploadState: + + def __init__(self, req): + self.identifier = req.META['QUERY_STRING'] + self.state = {'size': int(req.header_dict.get('content-length')), + 'state': 'starting', 'received': 0} + self.save() + + def addlen(self, toadd): + self.state['received'] = self.state['received'] + toadd + if self.state['size']-1 <= self.state['received']: + self.state['state'] = 'done' + else: + self.state['state'] = 'uploading' + self.save() + + def save(self): + simplejson.dump(self.state,open(get_temp_file(self.identifier), 'w')) + +class UploadStateMiddleware: + + def process_request(self, request): + + try: + progress_url=settings.PROGRESS_URL + except: + progress_url='/progress/' + + if request.META['QUERY_STRING']: + request.upload_state = UploadState + + if request.path == progress_url: + for header in request.header_dict.items(): + if header[0].upper().replace('-', '_').endswith('X_PROGRESS_ID'): + progress_id = header[1] + try: + content = open(get_temp_file(progress_id), 'r').read() + except: + content="{}" + if not content: + content="{}" + + from django.http import HttpResponse + return HttpResponse(content=content, mimetype='text/plain')