Ticket #2070: 3581-streaming_uploads_and_uploadprogress_middleware_x_progress_id_windowsfix.diff

File 3581-streaming_uploads_and_uploadprogress_middleware_x_progress_id_windowsfix.diff, 16.9 KB (added by [Cha0S], 8 years ago)

windows temporary file locking fix

  • django/http/__init__.py

     
    11import os
     2import cgi
    23from Cookie import SimpleCookie
    34from pprint import pformat
    45from urllib import urlencode, quote
     
    4243    def is_secure(self):
    4344        return os.environ.get("HTTPS") == "on"
    4445
    45 def parse_file_upload(header_dict, post_data):
     46def default_parse_file_upload(req):
    4647    "Returns a tuple of (POST MultiValueDict, FILES MultiValueDict)"
    4748    import email, email.Message
    4849    from cgi import parse_header
    49     raw_message = '\r\n'.join(['%s:%s' % pair for pair in header_dict.items()])
    50     raw_message += '\r\n\r\n' + post_data
     50    raw_message = '\r\n'.join(['%s:%s' % pair for pair in req.header_dict.items()])
     51    raw_message += '\r\n\r\n' + req.raw_post_data
    5152    msg = email.message_from_string(raw_message)
    5253    POST = MultiValueDict()
    5354    FILES = MultiValueDict()
     
    7374                POST.appendlist(name_dict['name'], submessage.get_payload())
    7475    return POST, FILES
    7576
     77def parse_file_upload(req):
     78    "Returns a tuple of (POST MultiValueDict, FILES MultiValueDict)"
     79
     80    if hasattr(req, 'parse_file_upload'):
     81        return req.parse_file_upload(req)
     82
     83    return default_parse_file_upload(req)
     84
    7685class QueryDict(MultiValueDict):
    7786    """A specialized MultiValueDict that takes a query string when initialized.
    7887    This is immutable unless you create a copy of it."""
  • django/db/models/base.py

     
    322322    def _get_FIELD_size(self, field):
    323323        return os.path.getsize(self._get_FIELD_filename(field))
    324324
    325     def _save_FIELD_file(self, field, filename, raw_contents):
     325    def _save_FIELD_file(self, field, filename, raw_field):
    326326        directory = field.get_directory_name()
    327327        try: # Create the date-based directory if it doesn't exist.
    328328            os.makedirs(os.path.join(settings.MEDIA_ROOT, directory))
     
    344344        setattr(self, field.attname, filename)
    345345
    346346        full_filename = self._get_FIELD_filename(field)
    347         fp = open(full_filename, 'wb')
    348         fp.write(raw_contents)
    349         fp.close()
    350347
     348        if not hasattr(raw_field, 'get_size'):
     349            fp = open(full_filename, 'wb')
     350            fp.write(raw_field['content'])
     351            fp.close()
     352        else:
     353            fp = raw_field['file']
     354            fp.seek(0)
     355            fdst = open(full_filename, 'wb')
     356            while 1:
     357                buf = fp.read(16384)
     358                if not buf:
     359                    break
     360                fdst.write(buf)
     361            fdst.close()
     362
    351363        # Save the width and/or height, if applicable.
    352364        if isinstance(field, ImageField) and (field.width_field or field.height_field):
    353365            from django.utils.images import get_image_dimensions
  • django/db/models/fields/__init__.py

     
    600600        if new_data.get(upload_field_name, False):
    601601            func = getattr(new_object, 'save_%s_file' % self.name)
    602602            if rel:
    603                 func(new_data[upload_field_name][0]["filename"], new_data[upload_field_name][0]["content"])
     603                func(new_data[upload_field_name][0]["filename"], new_data[upload_field_name][0])
    604604            else:
    605                 func(new_data[upload_field_name]["filename"], new_data[upload_field_name]["content"])
     605                func(new_data[upload_field_name]["filename"], new_data[upload_field_name])
    606606
    607607    def get_directory_name(self):
    608608        return os.path.normpath(datetime.datetime.now().strftime(self.upload_to))
  • django/forms/__init__.py

     
    655655        self.validator_list = [self.isNonEmptyFile] + validator_list
    656656
    657657    def isNonEmptyFile(self, field_data, all_data):
     658
    658659        try:
    659             content = field_data['content']
    660         except TypeError:
    661             raise validators.CriticalValidationError, gettext("No file was submitted. Check the encoding type on the form.")
     660            content = field_data.get_size()
     661
     662        except:
     663            try:
     664                content = field_data['content']
     665            except TypeError:
     666                raise validators.CriticalValidationError, gettext("No file was submitted. Check the encoding type on the form.")
     667
    662668        if not content:
    663669            raise validators.CriticalValidationError, gettext("The submitted file is empty.")
    664670
  • django/core/handlers/wsgi.py

     
    7272        # Populates self._post and self._files
    7373        if self.method == 'POST':
    7474            if self.environ.get('CONTENT_TYPE', '').startswith('multipart'):
    75                 header_dict = dict([(k, v) for k, v in self.environ.items() if k.startswith('HTTP_')])
    76                 header_dict['Content-Type'] = self.environ.get('CONTENT_TYPE', '')
    77                 self._post, self._files = http.parse_file_upload(header_dict, self.raw_post_data)
     75                self._post, self._files = http.parse_file_upload(self)
    7876            else:
    7977                self._post, self._files = http.QueryDict(self.raw_post_data), datastructures.MultiValueDict()
    8078        else:
     
    121119        except AttributeError:
    122120            self._raw_post_data = self.environ['wsgi.input'].read(int(self.environ["CONTENT_LENGTH"]))
    123121            return self._raw_post_data
     122   
     123    def _get_raw_request(self):
     124        return self.environ['wsgi.input']
    124125
     126    def _get_header_dict(self):
     127        header_dict = dict([(k, v) for k, v in self.environ.items() if k.startswith('HTTP_')])
     128        header_dict['Content-Type'] = self.environ.get('CONTENT_TYPE', '')
     129        return header_dict
     130
    125131    GET = property(_get_get, _set_get)
    126132    POST = property(_get_post, _set_post)
    127133    COOKIES = property(_get_cookies, _set_cookies)
    128134    FILES = property(_get_files)
    129135    REQUEST = property(_get_request)
    130136    raw_post_data = property(_get_raw_post_data)
     137    raw_request = property(_get_raw_request)
     138    header_dict = property(_get_header_dict)
    131139
    132140class WSGIHandler(BaseHandler):
    133141    def __call__(self, environ, start_response):
  • django/core/handlers/modpython.py

     
    2929    def _load_post_and_files(self):
    3030        "Populates self._post and self._files"
    3131        if self._req.headers_in.has_key('content-type') and self._req.headers_in['content-type'].startswith('multipart'):
    32             self._post, self._files = http.parse_file_upload(self._req.headers_in, self.raw_post_data)
     32            self._post, self._files = http.parse_file_upload(self)
    3333        else:
    3434            self._post, self._files = http.QueryDict(self.raw_post_data), datastructures.MultiValueDict()
    3535
     
    104104    def _get_method(self):
    105105        return self.META['REQUEST_METHOD'].upper()
    106106
     107    def _get_raw_request(self):
     108        return self._req
     109
     110    def _get_header_dict(self):
     111        return self._req.headers_in
     112
    107113    GET = property(_get_get, _set_get)
    108114    POST = property(_get_post, _set_post)
    109115    COOKIES = property(_get_cookies, _set_cookies)
     
    112118    REQUEST = property(_get_request)
    113119    raw_post_data = property(_get_raw_post_data)
    114120    method = property(_get_method)
     121    raw_request = property(_get_raw_request)
     122    header_dict = property(_get_header_dict)   
    115123
    116124class ModPythonHandler(BaseHandler):
    117125    def __call__(self, req):
  • django/contrib/admin/media/js/UploadProgress.js

     
     1
     2function getxy(){
     3    var x,y;
     4    if (self.innerHeight) // all except Explorer
     5        {
     6        x = self.innerWidth;
     7        y = self.innerHeight;
     8        }
     9    else if (document.documentElement && document.documentElement.clientHeight)
     10        // Explorer 6 Strict Mode
     11        {
     12        x = document.documentElement.clientWidth;
     13        y = document.documentElement.clientHeight;
     14        }
     15    else if (document.body) // other Explorers
     16        {
     17        x = document.body.clientWidth;
     18        y = document.body.clientHeight;
     19        }
     20    return {'x':x,'y':y}
     21    }
     22
     23var humanvalue = ['B','KB','MB','GB']
     24function humanize(bytes) {
     25    curbytes = bytes
     26    iterations = 0
     27    while (curbytes>1024) {
     28        iterations++
     29        curbytes=curbytes/1024
     30        }
     31    return curbytes.toFixed(1) + ' ' + humanvalue[iterations]
     32    }
     33
     34interval = null;
     35function fetch(uuid) {
     36    req = xmlhttp
     37    req.open("GET", "/progress/", 1);
     38    req.setRequestHeader("X-Progress-Id", uuid);
     39    req.onreadystatechange = function () {
     40    if (req.readyState == 4) {
     41        if (req.status == 200) {
     42
     43            var upload = eval( '(' + req.responseText + ')' );
     44
     45            if (upload.state == 'done' || upload.state == 'uploading') {
     46                bar = document.getElementById('progress_bar');
     47                bar_txt = document.getElementById('progress_text')
     48                bar_txt.innerHTML = ((upload.received / upload.size) * 100).toFixed(1) + '% - ' +
     49                    humanize(upload.received) + ' of ' + humanize(upload.size)
     50                w = 400 * upload.received / upload.size;
     51                bar.style.width = w + 'px';
     52
     53                }
     54                if (upload.state == 'done') {
     55                    window.clearTimeout(interval);
     56                    }
     57                }
     58            }
     59        }
     60    req.send(null);
     61
     62    }
     63
     64function openprogress(e) {
     65
     66    uuid = "";
     67    for (i = 0; i < 32; i++) {
     68        uuid += Math.floor(Math.random() * 16).toString(16);
     69        }
     70    frm = e.target||e.srcElement
     71
     72    frm.action=frm.action+"?" + uuid;
     73
     74    pos = getxy()
     75    posx = parseInt((pos.x/2)-(420/2), 10)
     76    posy = parseInt((pos.y/2)-(50/2), 10)
     77
     78    progress_wrap = quickElement('div', document.body, '', 'style',
     79        'position: absolute; top: '+posy+'px; left: '+posx+'px; height: 50px; ' +
     80        'padding: 10px; width: 420px; background: #ffffff; ' +
     81        'border: solid 1px #dddddd;', 'id', 'progress_wrap')
     82
     83    progress_label = quickElement('h1', progress_wrap, 'Upload progress')
     84
     85    progress = quickElement('div', progress_wrap, '', 'style',
     86        'top: 0; left: 0; width: 0px; ', 'id', 'progress_bar', 'class', 'submit-row')
     87
     88    progress_text = quickElement('div', progress_wrap, '0%', 'style',
     89        'color: #000000; ', 'id', 'progress_text')
     90 
     91    interval = window.setInterval(
     92        function () {
     93            fetch(uuid);
     94            },
     95        1000
     96        );
     97    }
     98
     99addEvent(window, 'load', function() {
     100        frm = document.getElementsByTagName('form')[0]
     101        addEvent(frm, 'submit',  openprogress)   
     102        }
     103    )
  • django/middleware/upload.py

     
     1"streaming upload middleware"
     2import cgi
     3import os
     4import tempfile
     5from django.conf import settings
     6from django.utils.datastructures import MultiValueDict
     7from django.utils import simplejson
     8
     9try:
     10    UPLOAD_BUFFER_SIZE = settings.UPLOAD_BUFFER_SIZE
     11except:
     12    UPLOAD_BUFFER_SIZE = 64000
     13
     14class FileDict(dict):
     15    "Keeps uploaded file as a file-like object and reads its content on demand"
     16    def __getitem__(self, name):
     17        if name=='content' and not 'content' in self:
     18            self['file'].seek(0, 2)
     19            size = self['file'].tell()
     20            self['file'].seek(0, 0)
     21            self['content']=self['file'].read(size)
     22        return dict.__getitem__(self, name)
     23       
     24    def get_size(self):
     25        self['file'].seek(0, 2)   
     26        size = self['file'].tell()
     27        return size
     28
     29    def __repr__(self):
     30        return '<FileDict>'
     31
     32class PostStream:
     33
     34    def __init__(self, fp, upload_state=None):
     35        self.fp = fp
     36        self.upload_state = upload_state
     37        self.bufsize = UPLOAD_BUFFER_SIZE
     38 
     39    def readline(self):
     40        data = self.fp.readline(self.bufsize)
     41        if self.upload_state:
     42            self.upload_state.addlen(len(data))
     43        return data
     44
     45    def read(self):
     46        data = self.fp.read(self.bufsize)
     47        if self.upload_state:
     48            self.upload_state.addlen(len(data))
     49        return data
     50
     51class FieldStorageM(cgi.FieldStorage, object):
     52    pass
     53 
     54class FieldStorage(FieldStorageM):
     55
     56    upload_state = None
     57    bufsize = UPLOAD_BUFFER_SIZE
     58     
     59    "cgi.FieldStorage with ability to store files on disk"
     60
     61    def __init__(self, fp=None, headers=None, outerboundary="",
     62                 environ=os.environ, keep_blank_values=0, strict_parsing=0, upload_state=None):
     63
     64        if not isinstance(fp, PostStream):
     65            stream = PostStream(fp, upload_state=upload_state)
     66        else:
     67            stream = fp
     68
     69        super(FieldStorage, self).__init__(fp=stream, headers=headers,
     70            outerboundary=outerboundary, environ=environ,
     71            keep_blank_values=keep_blank_values, strict_parsing=strict_parsing)
     72
     73    def make_file(self, binary=None):
     74     
     75        tmpfile = tempfile.NamedTemporaryFile("w+b")
     76        self.tmp_name = tmpfile.name
     77        return tmpfile
     78
     79
     80def parse_streaming_file_upload(req):
     81    "Returns a tuple of (POST MultiValueDict, FILES MultiValueDict)"
     82    if hasattr(req, 'upload_state'):
     83        upload_state = req.upload_state(req)
     84    else:
     85        upload_state = None
     86
     87    fs = FieldStorage(req.raw_request, environ=req.META, headers=req.header_dict, upload_state=upload_state )
     88    POST = MultiValueDict()
     89    FILES = MultiValueDict()
     90    for key in fs.keys():
     91        # We can't use FieldStorage.getlist to get contents of a
     92        # field as a list because for file fields it returns only filenames
     93        if type(fs[key]) == type([]):
     94            field_list = fs[key]
     95        else:
     96            field_list = [fs[key]]
     97        for field in field_list:
     98            if hasattr(field, 'filename') and field.filename is not None:
     99                if not field.filename.strip():
     100                    continue
     101                # IE submits the full path, so trim everything but the basename.
     102                # (We can't use os.path.basename because it expects Linux paths.)
     103                filename = field.filename[field.filename.rfind("\\") + 1:]
     104                FILES.appendlist(key, FileDict({
     105                    'filename': filename,
     106                    'content-type': field.type,
     107                    'file': field.file,
     108                    'tmp_name': field.tmp_name
     109                }))
     110            else:
     111                POST.appendlist(key, field.value)
     112    return POST, FILES
     113
     114class StreamingUploadMiddleware:
     115
     116    def process_request(self, request):
     117        request.parse_file_upload = parse_streaming_file_upload
     118
     119def get_temp_file(identifier):
     120    return os.path.join(tempfile.gettempdir(),identifier)
     121
     122class UploadState:
     123
     124    def __init__(self, req):
     125        self.identifier = req.META['QUERY_STRING']
     126        self.state = {'size': int(req.header_dict.get('content-length')),
     127             'state': 'starting', 'received': 0}
     128        self.save()
     129
     130    def addlen(self, toadd):
     131        self.state['received'] = self.state['received'] + toadd
     132        if self.state['size']-1 <= self.state['received']:
     133            self.state['state'] = 'done'
     134        else:
     135             self.state['state'] = 'uploading'
     136        self.save()
     137
     138    def save(self):
     139        simplejson.dump(self.state,open(get_temp_file(self.identifier), 'w'))
     140
     141class UploadStateMiddleware:
     142    def process_request(self, request):
     143        if request.META['QUERY_STRING']:
     144            request.upload_state = UploadState
     145        if request.path == '/progress/':
     146            for header in request.header_dict.items():
     147                if header[0].upper().replace('-', '_').endswith('X_PROGRESS_ID'):
     148                    progress_id = header[1]
     149            try:
     150                content = open(get_temp_file(progress_id), 'r').read()
     151            except:
     152                content="{}"
     153            if not content:
     154                content="{}"
     155
     156            from django.http import HttpResponse
     157            return HttpResponse(content=content, mimetype='text/plain')
Back to Top