Index: django/http/__init__.py =================================================================== --- django/http/__init__.py (revision 3581) +++ django/http/__init__.py (working copy) @@ -1,4 +1,5 @@ import os +import cgi from Cookie import SimpleCookie from pprint import pformat from urllib import urlencode, quote @@ -42,12 +43,12 @@ def is_secure(self): return os.environ.get("HTTPS") == "on" -def parse_file_upload(header_dict, post_data): +def default_parse_file_upload(req): "Returns a tuple of (POST MultiValueDict, FILES MultiValueDict)" import email, email.Message from cgi import parse_header - raw_message = '\r\n'.join(['%s:%s' % pair for pair in header_dict.items()]) - raw_message += '\r\n\r\n' + post_data + raw_message = '\r\n'.join(['%s:%s' % pair for pair in req.header_dict.items()]) + raw_message += '\r\n\r\n' + req.raw_post_data msg = email.message_from_string(raw_message) POST = MultiValueDict() FILES = MultiValueDict() @@ -73,6 +74,14 @@ POST.appendlist(name_dict['name'], submessage.get_payload()) return POST, FILES +def parse_file_upload(req): + "Returns a tuple of (POST MultiValueDict, FILES MultiValueDict)" + + if hasattr(req, 'parse_file_upload'): + return req.parse_file_upload(req) + + return default_parse_file_upload(req) + class QueryDict(MultiValueDict): """A specialized MultiValueDict that takes a query string when initialized. This is immutable unless you create a copy of it.""" Index: django/db/models/base.py =================================================================== --- django/db/models/base.py (revision 3581) +++ django/db/models/base.py (working copy) @@ -322,7 +322,7 @@ def _get_FIELD_size(self, field): return os.path.getsize(self._get_FIELD_filename(field)) - def _save_FIELD_file(self, field, filename, raw_contents): + def _save_FIELD_file(self, field, filename, raw_field): directory = field.get_directory_name() try: # Create the date-based directory if it doesn't exist. os.makedirs(os.path.join(settings.MEDIA_ROOT, directory)) @@ -344,10 +344,22 @@ setattr(self, field.attname, filename) full_filename = self._get_FIELD_filename(field) - fp = open(full_filename, 'wb') - fp.write(raw_contents) - fp.close() + if not hasattr(raw_field, 'get_size'): + fp = open(full_filename, 'wb') + fp.write(raw_field['content']) + fp.close() + else: + fp = raw_field['file'] + fp.seek(0) + fdst = open(full_filename, 'wb') + while 1: + buf = fp.read(16384) + if not buf: + break + fdst.write(buf) + fdst.close() + # Save the width and/or height, if applicable. if isinstance(field, ImageField) and (field.width_field or field.height_field): from django.utils.images import get_image_dimensions Index: django/db/models/fields/__init__.py =================================================================== --- django/db/models/fields/__init__.py (revision 3581) +++ django/db/models/fields/__init__.py (working copy) @@ -600,9 +600,9 @@ if new_data.get(upload_field_name, False): func = getattr(new_object, 'save_%s_file' % self.name) if rel: - func(new_data[upload_field_name][0]["filename"], new_data[upload_field_name][0]["content"]) + func(new_data[upload_field_name][0]["filename"], new_data[upload_field_name][0]) else: - func(new_data[upload_field_name]["filename"], new_data[upload_field_name]["content"]) + func(new_data[upload_field_name]["filename"], new_data[upload_field_name]) def get_directory_name(self): return os.path.normpath(datetime.datetime.now().strftime(self.upload_to)) Index: django/forms/__init__.py =================================================================== --- django/forms/__init__.py (revision 3581) +++ django/forms/__init__.py (working copy) @@ -655,10 +655,16 @@ self.validator_list = [self.isNonEmptyFile] + validator_list def isNonEmptyFile(self, field_data, all_data): + try: - content = field_data['content'] - except TypeError: - raise validators.CriticalValidationError, gettext("No file was submitted. Check the encoding type on the form.") + content = field_data.get_size() + + except: + try: + content = field_data['content'] + except TypeError: + raise validators.CriticalValidationError, gettext("No file was submitted. Check the encoding type on the form.") + if not content: raise validators.CriticalValidationError, gettext("The submitted file is empty.") Index: django/core/handlers/wsgi.py =================================================================== --- django/core/handlers/wsgi.py (revision 3581) +++ django/core/handlers/wsgi.py (working copy) @@ -72,9 +72,7 @@ # Populates self._post and self._files if self.method == 'POST': if self.environ.get('CONTENT_TYPE', '').startswith('multipart'): - header_dict = dict([(k, v) for k, v in self.environ.items() if k.startswith('HTTP_')]) - header_dict['Content-Type'] = self.environ.get('CONTENT_TYPE', '') - self._post, self._files = http.parse_file_upload(header_dict, self.raw_post_data) + self._post, self._files = http.parse_file_upload(self) else: self._post, self._files = http.QueryDict(self.raw_post_data), datastructures.MultiValueDict() else: @@ -121,13 +119,23 @@ except AttributeError: self._raw_post_data = self.environ['wsgi.input'].read(int(self.environ["CONTENT_LENGTH"])) return self._raw_post_data + + def _get_raw_request(self): + return self.environ['wsgi.input'] + def _get_header_dict(self): + header_dict = dict([(k, v) for k, v in self.environ.items() if k.startswith('HTTP_')]) + header_dict['Content-Type'] = self.environ.get('CONTENT_TYPE', '') + return header_dict + GET = property(_get_get, _set_get) POST = property(_get_post, _set_post) COOKIES = property(_get_cookies, _set_cookies) FILES = property(_get_files) REQUEST = property(_get_request) raw_post_data = property(_get_raw_post_data) + raw_request = property(_get_raw_request) + header_dict = property(_get_header_dict) class WSGIHandler(BaseHandler): def __call__(self, environ, start_response): Index: django/core/handlers/modpython.py =================================================================== --- django/core/handlers/modpython.py (revision 3581) +++ django/core/handlers/modpython.py (working copy) @@ -29,7 +29,7 @@ def _load_post_and_files(self): "Populates self._post and self._files" if self._req.headers_in.has_key('content-type') and self._req.headers_in['content-type'].startswith('multipart'): - self._post, self._files = http.parse_file_upload(self._req.headers_in, self.raw_post_data) + self._post, self._files = http.parse_file_upload(self) else: self._post, self._files = http.QueryDict(self.raw_post_data), datastructures.MultiValueDict() @@ -104,6 +104,12 @@ def _get_method(self): return self.META['REQUEST_METHOD'].upper() + def _get_raw_request(self): + return self._req + + def _get_header_dict(self): + return self._req.headers_in + GET = property(_get_get, _set_get) POST = property(_get_post, _set_post) COOKIES = property(_get_cookies, _set_cookies) @@ -112,6 +118,8 @@ REQUEST = property(_get_request) raw_post_data = property(_get_raw_post_data) method = property(_get_method) + raw_request = property(_get_raw_request) + header_dict = property(_get_header_dict) class ModPythonHandler(BaseHandler): def __call__(self, req): Index: django/contrib/admin/media/js/UploadProgress.js =================================================================== --- django/contrib/admin/media/js/UploadProgress.js (revision 0) +++ django/contrib/admin/media/js/UploadProgress.js (revision 0) @@ -0,0 +1,103 @@ + +function getxy(){ + var x,y; + if (self.innerHeight) // all except Explorer + { + x = self.innerWidth; + y = self.innerHeight; + } + else if (document.documentElement && document.documentElement.clientHeight) + // Explorer 6 Strict Mode + { + x = document.documentElement.clientWidth; + y = document.documentElement.clientHeight; + } + else if (document.body) // other Explorers + { + x = document.body.clientWidth; + y = document.body.clientHeight; + } + return {'x':x,'y':y} + } + +var humanvalue = ['B','KB','MB','GB'] +function humanize(bytes) { + curbytes = bytes + iterations = 0 + while (curbytes>1024) { + iterations++ + curbytes=curbytes/1024 + } + return curbytes.toFixed(1) + ' ' + humanvalue[iterations] + } + +interval = null; +function fetch(uuid) { + req = xmlhttp + req.open("GET", "/progress/", 1); + req.setRequestHeader("X-Progress-Id", uuid); + req.onreadystatechange = function () { + if (req.readyState == 4) { + if (req.status == 200) { + + var upload = eval( '(' + req.responseText + ')' ); + + if (upload.state == 'done' || upload.state == 'uploading') { + bar = document.getElementById('progress_bar'); + bar_txt = document.getElementById('progress_text') + bar_txt.innerHTML = ((upload.received / upload.size) * 100).toFixed(1) + '% - ' + + humanize(upload.received) + ' of ' + humanize(upload.size) + w = 400 * upload.received / upload.size; + bar.style.width = w + 'px'; + + } + if (upload.state == 'done') { + window.clearTimeout(interval); + } + } + } + } + req.send(null); + + } + +function openprogress(e) { + + uuid = ""; + for (i = 0; i < 32; i++) { + uuid += Math.floor(Math.random() * 16).toString(16); + } + frm = e.target||e.srcElement + + frm.action=frm.action+"?" + uuid; + + pos = getxy() + posx = parseInt((pos.x/2)-(420/2), 10) + posy = parseInt((pos.y/2)-(50/2), 10) + + progress_wrap = quickElement('div', document.body, '', 'style', + 'position: absolute; top: '+posy+'px; left: '+posx+'px; height: 50px; ' + + 'padding: 10px; width: 420px; background: #ffffff; ' + + 'border: solid 1px #dddddd;', 'id', 'progress_wrap') + + progress_label = quickElement('h1', progress_wrap, 'Upload progress') + + progress = quickElement('div', progress_wrap, '', 'style', + 'top: 0; left: 0; width: 0px; ', 'id', 'progress_bar', 'class', 'submit-row') + + progress_text = quickElement('div', progress_wrap, '0%', 'style', + 'color: #000000; ', 'id', 'progress_text') + + interval = window.setInterval( + function () { + fetch(uuid); + }, + 1000 + ); + } + +addEvent(window, 'load', function() { + frm = document.getElementsByTagName('form')[0] + addEvent(frm, 'submit', openprogress) + } + ) Index: django/middleware/upload.py =================================================================== --- django/middleware/upload.py (revision 0) +++ django/middleware/upload.py (revision 0) @@ -0,0 +1,157 @@ +"streaming upload middleware" +import cgi +import os +import tempfile +from django.conf import settings +from django.utils.datastructures import MultiValueDict +from django.utils import simplejson + +try: + UPLOAD_BUFFER_SIZE = settings.UPLOAD_BUFFER_SIZE +except: + UPLOAD_BUFFER_SIZE = 64000 + +class FileDict(dict): + "Keeps uploaded file as a file-like object and reads its content on demand" + def __getitem__(self, name): + if name=='content' and not 'content' in self: + self['file'].seek(0, 2) + size = self['file'].tell() + self['file'].seek(0, 0) + self['content']=self['file'].read(size) + return dict.__getitem__(self, name) + + def get_size(self): + self['file'].seek(0, 2) + size = self['file'].tell() + return size + + def __repr__(self): + return '' + +class PostStream: + + def __init__(self, fp, upload_state=None): + self.fp = fp + self.upload_state = upload_state + self.bufsize = UPLOAD_BUFFER_SIZE + + def readline(self): + data = self.fp.readline(self.bufsize) + if self.upload_state: + self.upload_state.addlen(len(data)) + return data + + def read(self): + data = self.fp.read(self.bufsize) + if self.upload_state: + self.upload_state.addlen(len(data)) + return data + +class FieldStorageM(cgi.FieldStorage, object): + pass + +class FieldStorage(FieldStorageM): + + upload_state = None + bufsize = UPLOAD_BUFFER_SIZE + + "cgi.FieldStorage with ability to store files on disk" + + def __init__(self, fp=None, headers=None, outerboundary="", + environ=os.environ, keep_blank_values=0, strict_parsing=0, upload_state=None): + + if not isinstance(fp, PostStream): + stream = PostStream(fp, upload_state=upload_state) + else: + stream = fp + + super(FieldStorage, self).__init__(fp=stream, headers=headers, + outerboundary=outerboundary, environ=environ, + keep_blank_values=keep_blank_values, strict_parsing=strict_parsing) + + def make_file(self, binary=None): + + tmpfile = tempfile.NamedTemporaryFile("w+b") + self.tmp_name = tmpfile.name + return tmpfile + + +def parse_streaming_file_upload(req): + "Returns a tuple of (POST MultiValueDict, FILES MultiValueDict)" + if hasattr(req, 'upload_state'): + upload_state = req.upload_state(req) + else: + upload_state = None + + fs = FieldStorage(req.raw_request, environ=req.META, headers=req.header_dict, upload_state=upload_state ) + POST = MultiValueDict() + FILES = MultiValueDict() + for key in fs.keys(): + # We can't use FieldStorage.getlist to get contents of a + # field as a list because for file fields it returns only filenames + if type(fs[key]) == type([]): + field_list = fs[key] + else: + field_list = [fs[key]] + for field in field_list: + if hasattr(field, 'filename') and field.filename is not None: + if not field.filename.strip(): + continue + # IE submits the full path, so trim everything but the basename. + # (We can't use os.path.basename because it expects Linux paths.) + filename = field.filename[field.filename.rfind("\\") + 1:] + FILES.appendlist(key, FileDict({ + 'filename': filename, + 'content-type': field.type, + 'file': field.file, + 'tmp_name': field.tmp_name + })) + else: + POST.appendlist(key, field.value) + return POST, FILES + +class StreamingUploadMiddleware: + + def process_request(self, request): + request.parse_file_upload = parse_streaming_file_upload + +def get_temp_file(identifier): + return os.path.join(tempfile.gettempdir(),identifier) + +class UploadState: + + def __init__(self, req): + self.identifier = req.META['QUERY_STRING'] + self.state = {'size': int(req.header_dict.get('content-length')), + 'state': 'starting', 'received': 0} + self.save() + + def addlen(self, toadd): + self.state['received'] = self.state['received'] + toadd + if self.state['size']-1 <= self.state['received']: + self.state['state'] = 'done' + else: + self.state['state'] = 'uploading' + self.save() + + def save(self): + simplejson.dump(self.state,open(get_temp_file(self.identifier), 'w')) + +class UploadStateMiddleware: + def process_request(self, request): + if request.META['QUERY_STRING']: + request.upload_state = UploadState + if request.path == '/progress/': + for header in request.header_dict.items(): + if header[0].upper().replace('-', '_').endswith('X_PROGRESS_ID'): + progress_id = header[1] + try: + content = open(get_temp_file(progress_id), 'r').read() + except: + content="{}" + if not content: + content="{}" + + from django.http import HttpResponse + return HttpResponse(content=content, mimetype='text/plain')