Code

Ticket #2070: 3581-streaming_uploads_and_uploadprogress_middleware_x_progress_id.diff

File 3581-streaming_uploads_and_uploadprogress_middleware_x_progress_id.diff, 16.7 KB (added by [530], 8 years ago)

Now using X_PROGRESS_ID instead of X-Progress-Id, accepts any lower/uppercase/ - / _ /prefix variant

Line 
1Index: django/http/__init__.py
2===================================================================
3--- django/http/__init__.py     (revision 3581)
4+++ django/http/__init__.py     (working copy)
5@@ -1,4 +1,5 @@
6 import os
7+import cgi
8 from Cookie import SimpleCookie
9 from pprint import pformat
10 from urllib import urlencode, quote
11@@ -42,12 +43,12 @@
12     def is_secure(self):
13         return os.environ.get("HTTPS") == "on"
14 
15-def parse_file_upload(header_dict, post_data):
16+def default_parse_file_upload(req):
17     "Returns a tuple of (POST MultiValueDict, FILES MultiValueDict)"
18     import email, email.Message
19     from cgi import parse_header
20-    raw_message = '\r\n'.join(['%s:%s' % pair for pair in header_dict.items()])
21-    raw_message += '\r\n\r\n' + post_data
22+    raw_message = '\r\n'.join(['%s:%s' % pair for pair in req.header_dict.items()])
23+    raw_message += '\r\n\r\n' + req.raw_post_data
24     msg = email.message_from_string(raw_message)
25     POST = MultiValueDict()
26     FILES = MultiValueDict()
27@@ -73,6 +74,14 @@
28                 POST.appendlist(name_dict['name'], submessage.get_payload())
29     return POST, FILES
30 
31+def parse_file_upload(req):
32+    "Returns a tuple of (POST MultiValueDict, FILES MultiValueDict)"
33+
34+    if hasattr(req, 'parse_file_upload'):
35+        return req.parse_file_upload(req)
36+
37+    return default_parse_file_upload(req)
38+
39 class QueryDict(MultiValueDict):
40     """A specialized MultiValueDict that takes a query string when initialized.
41     This is immutable unless you create a copy of it."""
42Index: django/db/models/base.py
43===================================================================
44--- django/db/models/base.py    (revision 3581)
45+++ django/db/models/base.py    (working copy)
46@@ -322,7 +322,7 @@
47     def _get_FIELD_size(self, field):
48         return os.path.getsize(self._get_FIELD_filename(field))
49 
50-    def _save_FIELD_file(self, field, filename, raw_contents):
51+    def _save_FIELD_file(self, field, filename, raw_field):
52         directory = field.get_directory_name()
53         try: # Create the date-based directory if it doesn't exist.
54             os.makedirs(os.path.join(settings.MEDIA_ROOT, directory))
55@@ -344,10 +344,14 @@
56         setattr(self, field.attname, filename)
57 
58         full_filename = self._get_FIELD_filename(field)
59-        fp = open(full_filename, 'wb')
60-        fp.write(raw_contents)
61-        fp.close()
62 
63+        if not hasattr(raw_field, 'get_size'):
64+            fp = open(full_filename, 'wb')
65+            fp.write(raw_field['content'])
66+            fp.close()
67+        else:
68+            os.rename(raw_field['tmp_name'], full_filename)
69+
70         # Save the width and/or height, if applicable.
71         if isinstance(field, ImageField) and (field.width_field or field.height_field):
72             from django.utils.images import get_image_dimensions
73Index: django/db/models/fields/__init__.py
74===================================================================
75--- django/db/models/fields/__init__.py (revision 3581)
76+++ django/db/models/fields/__init__.py (working copy)
77@@ -600,9 +600,9 @@
78         if new_data.get(upload_field_name, False):
79             func = getattr(new_object, 'save_%s_file' % self.name)
80             if rel:
81-                func(new_data[upload_field_name][0]["filename"], new_data[upload_field_name][0]["content"])
82+                func(new_data[upload_field_name][0]["filename"], new_data[upload_field_name][0])
83             else:
84-                func(new_data[upload_field_name]["filename"], new_data[upload_field_name]["content"])
85+                func(new_data[upload_field_name]["filename"], new_data[upload_field_name])
86 
87     def get_directory_name(self):
88         return os.path.normpath(datetime.datetime.now().strftime(self.upload_to))
89Index: django/forms/__init__.py
90===================================================================
91--- django/forms/__init__.py    (revision 3581)
92+++ django/forms/__init__.py    (working copy)
93@@ -655,10 +655,16 @@
94         self.validator_list = [self.isNonEmptyFile] + validator_list
95 
96     def isNonEmptyFile(self, field_data, all_data):
97+
98         try:
99-            content = field_data['content']
100-        except TypeError:
101-            raise validators.CriticalValidationError, gettext("No file was submitted. Check the encoding type on the form.")
102+            content = field_data.get_size()
103+
104+        except:
105+            try:
106+                content = field_data['content']
107+            except TypeError:
108+                raise validators.CriticalValidationError, gettext("No file was submitted. Check the encoding type on the form.")
109+
110         if not content:
111             raise validators.CriticalValidationError, gettext("The submitted file is empty.")
112 
113Index: django/core/handlers/wsgi.py
114===================================================================
115--- django/core/handlers/wsgi.py        (revision 3581)
116+++ django/core/handlers/wsgi.py        (working copy)
117@@ -72,9 +72,7 @@
118         # Populates self._post and self._files
119         if self.method == 'POST':
120             if self.environ.get('CONTENT_TYPE', '').startswith('multipart'):
121-                header_dict = dict([(k, v) for k, v in self.environ.items() if k.startswith('HTTP_')])
122-                header_dict['Content-Type'] = self.environ.get('CONTENT_TYPE', '')
123-                self._post, self._files = http.parse_file_upload(header_dict, self.raw_post_data)
124+                self._post, self._files = http.parse_file_upload(self)
125             else:
126                 self._post, self._files = http.QueryDict(self.raw_post_data), datastructures.MultiValueDict()
127         else:
128@@ -121,13 +119,23 @@
129         except AttributeError:
130             self._raw_post_data = self.environ['wsgi.input'].read(int(self.environ["CONTENT_LENGTH"]))
131             return self._raw_post_data
132+   
133+    def _get_raw_request(self):
134+        return self.environ['wsgi.input']
135 
136+    def _get_header_dict(self):
137+        header_dict = dict([(k, v) for k, v in self.environ.items() if k.startswith('HTTP_')])
138+        header_dict['Content-Type'] = self.environ.get('CONTENT_TYPE', '')
139+        return header_dict
140+
141     GET = property(_get_get, _set_get)
142     POST = property(_get_post, _set_post)
143     COOKIES = property(_get_cookies, _set_cookies)
144     FILES = property(_get_files)
145     REQUEST = property(_get_request)
146     raw_post_data = property(_get_raw_post_data)
147+    raw_request = property(_get_raw_request)
148+    header_dict = property(_get_header_dict)
149 
150 class WSGIHandler(BaseHandler):
151     def __call__(self, environ, start_response):
152Index: django/core/handlers/modpython.py
153===================================================================
154--- django/core/handlers/modpython.py   (revision 3581)
155+++ django/core/handlers/modpython.py   (working copy)
156@@ -29,7 +29,7 @@
157     def _load_post_and_files(self):
158         "Populates self._post and self._files"
159         if self._req.headers_in.has_key('content-type') and self._req.headers_in['content-type'].startswith('multipart'):
160-            self._post, self._files = http.parse_file_upload(self._req.headers_in, self.raw_post_data)
161+            self._post, self._files = http.parse_file_upload(self)
162         else:
163             self._post, self._files = http.QueryDict(self.raw_post_data), datastructures.MultiValueDict()
164 
165@@ -104,6 +104,12 @@
166     def _get_method(self):
167         return self.META['REQUEST_METHOD'].upper()
168 
169+    def _get_raw_request(self):
170+        return self._req
171+
172+    def _get_header_dict(self):
173+        return self._req.headers_in
174+
175     GET = property(_get_get, _set_get)
176     POST = property(_get_post, _set_post)
177     COOKIES = property(_get_cookies, _set_cookies)
178@@ -112,6 +118,8 @@
179     REQUEST = property(_get_request)
180     raw_post_data = property(_get_raw_post_data)
181     method = property(_get_method)
182+    raw_request = property(_get_raw_request)
183+    header_dict = property(_get_header_dict)   
184 
185 class ModPythonHandler(BaseHandler):
186     def __call__(self, req):
187Index: django/contrib/admin/media/js/UploadProgress.js
188===================================================================
189--- django/contrib/admin/media/js/UploadProgress.js     (revision 0)
190+++ django/contrib/admin/media/js/UploadProgress.js     (revision 0)
191@@ -0,0 +1,103 @@
192+
193+function getxy(){
194+    var x,y;
195+    if (self.innerHeight) // all except Explorer
196+        {
197+        x = self.innerWidth;
198+        y = self.innerHeight;
199+        }
200+    else if (document.documentElement && document.documentElement.clientHeight)
201+        // Explorer 6 Strict Mode
202+        {
203+        x = document.documentElement.clientWidth;
204+        y = document.documentElement.clientHeight;
205+        }
206+    else if (document.body) // other Explorers
207+        {
208+        x = document.body.clientWidth;
209+        y = document.body.clientHeight;
210+        }
211+    return {'x':x,'y':y}
212+    }
213+
214+var humanvalue = ['B','KB','MB','GB']
215+function humanize(bytes) {
216+    curbytes = bytes
217+    iterations = 0
218+    while (curbytes>1024) {
219+        iterations++
220+        curbytes=curbytes/1024
221+        }
222+    return curbytes.toFixed(1) + ' ' + humanvalue[iterations]
223+    }
224+
225+interval = null;
226+function fetch(uuid) {
227+    req = xmlhttp
228+    req.open("GET", "/progress/", 1);
229+    req.setRequestHeader("X-Progress-Id", uuid);
230+    req.onreadystatechange = function () {
231+    if (req.readyState == 4) {
232+        if (req.status == 200) {
233+
234+            var upload = eval( '(' + req.responseText + ')' );
235+
236+            if (upload.state == 'done' || upload.state == 'uploading') {
237+                bar = document.getElementById('progress_bar');
238+                bar_txt = document.getElementById('progress_text')
239+                bar_txt.innerHTML = ((upload.received / upload.size) * 100).toFixed(1) + '% - ' +
240+                    humanize(upload.received) + ' of ' + humanize(upload.size)
241+                w = 400 * upload.received / upload.size;
242+                bar.style.width = w + 'px';
243+
244+                }
245+                if (upload.state == 'done') {
246+                    window.clearTimeout(interval);
247+                    }
248+                }
249+            }
250+        }
251+    req.send(null);
252+
253+    }
254+
255+function openprogress(e) {
256+
257+    uuid = "";
258+    for (i = 0; i < 32; i++) {
259+        uuid += Math.floor(Math.random() * 16).toString(16);
260+        }
261+    frm = e.target||e.srcElement
262+
263+    frm.action=frm.action+"?" + uuid;
264+
265+    pos = getxy()
266+    posx = parseInt((pos.x/2)-(420/2), 10)
267+    posy = parseInt((pos.y/2)-(50/2), 10)
268+
269+    progress_wrap = quickElement('div', document.body, '', 'style',
270+        'position: absolute; top: '+posy+'px; left: '+posx+'px; height: 50px; ' +
271+        'padding: 10px; width: 420px; background: #ffffff; ' +
272+        'border: solid 1px #dddddd;', 'id', 'progress_wrap')
273+
274+    progress_label = quickElement('h1', progress_wrap, 'Upload progress')
275+
276+    progress = quickElement('div', progress_wrap, '', 'style',
277+        'top: 0; left: 0; width: 0px; ', 'id', 'progress_bar', 'class', 'submit-row')
278+
279+    progress_text = quickElement('div', progress_wrap, '0%', 'style',
280+        'color: #000000; ', 'id', 'progress_text')
281+
282+    interval = window.setInterval(
283+        function () {
284+            fetch(uuid);
285+            },
286+        1000
287+        );
288+    }
289+
290+addEvent(window, 'load', function() {
291+        frm = document.getElementsByTagName('form')[0]
292+        addEvent(frm, 'submit',  openprogress)   
293+        }
294+    )
295Index: django/middleware/upload.py
296===================================================================
297--- django/middleware/upload.py (revision 0)
298+++ django/middleware/upload.py (revision 0)
299@@ -0,0 +1,157 @@
300+"streaming upload middleware"
301+import cgi
302+import os
303+import tempfile
304+from django.conf import settings
305+from django.utils.datastructures import MultiValueDict
306+from django.utils import simplejson
307+
308+try:
309+    UPLOAD_BUFFER_SIZE = settings.UPLOAD_BUFFER_SIZE
310+except:
311+    UPLOAD_BUFFER_SIZE = 64000
312+
313+class FileDict(dict):
314+    "Keeps uploaded file as a file-like object and reads its content on demand"
315+    def __getitem__(self, name):
316+        if name=='content' and not 'content' in self:
317+            self['file'].seek(0, 2)
318+            size = self['file'].tell()
319+            self['file'].seek(0, 0)
320+            self['content']=self['file'].read(size)
321+        return dict.__getitem__(self, name)
322+       
323+    def get_size(self):
324+        self['file'].seek(0, 2)   
325+        size = self['file'].tell()
326+        return size
327+
328+    def __repr__(self):
329+        return '<FileDict>'
330+
331+class PostStream:
332+
333+    def __init__(self, fp, upload_state=None):
334+        self.fp = fp
335+        self.upload_state = upload_state
336+        self.bufsize = UPLOAD_BUFFER_SIZE
337+
338+    def readline(self):
339+        data = self.fp.readline(self.bufsize)
340+        if self.upload_state:
341+            self.upload_state.addlen(len(data))
342+        return data
343+
344+    def read(self):
345+        data = self.fp.read(self.bufsize)
346+        if self.upload_state:
347+            self.upload_state.addlen(len(data))
348+        return data
349+
350+class FieldStorageM(cgi.FieldStorage, object):
351+    pass
352+
353+class FieldStorage(FieldStorageM):
354+
355+    upload_state = None
356+    bufsize = UPLOAD_BUFFER_SIZE
357+     
358+    "cgi.FieldStorage with ability to store files on disk"
359+
360+    def __init__(self, fp=None, headers=None, outerboundary="",
361+                 environ=os.environ, keep_blank_values=0, strict_parsing=0, upload_state=None):
362+
363+        if not isinstance(fp, PostStream):
364+            stream = PostStream(fp, upload_state=upload_state)
365+        else:
366+            stream = fp
367+
368+        super(FieldStorage, self).__init__(fp=stream, headers=headers,
369+            outerboundary=outerboundary, environ=environ,
370+            keep_blank_values=keep_blank_values, strict_parsing=strict_parsing)
371+
372+    def make_file(self, binary=None):
373+     
374+        tmpfile = tempfile.NamedTemporaryFile("w+b")
375+        self.tmp_name = tmpfile.name
376+        return tmpfile
377+
378+
379+def parse_streaming_file_upload(req):
380+    "Returns a tuple of (POST MultiValueDict, FILES MultiValueDict)"
381+    if hasattr(req, 'upload_state'):
382+        upload_state = req.upload_state(req)
383+    else:
384+        upload_state = None
385+
386+    fs = FieldStorage(req.raw_request, environ=req.META, headers=req.header_dict, upload_state=upload_state )
387+    POST = MultiValueDict()
388+    FILES = MultiValueDict()
389+    for key in fs.keys():
390+        # We can't use FieldStorage.getlist to get contents of a
391+        # field as a list because for file fields it returns only filenames
392+        if type(fs[key]) == type([]):
393+            field_list = fs[key]
394+        else:
395+            field_list = [fs[key]]
396+        for field in field_list:
397+            if hasattr(field, 'filename') and field.filename is not None:
398+                if not field.filename.strip():
399+                    continue
400+                # IE submits the full path, so trim everything but the basename.
401+                # (We can't use os.path.basename because it expects Linux paths.)
402+                filename = field.filename[field.filename.rfind("\\") + 1:]
403+                FILES.appendlist(key, FileDict({
404+                    'filename': filename,
405+                    'content-type': field.type,
406+                    'file': field.file,
407+                    'tmp_name': field.tmp_name
408+                }))
409+            else:
410+                POST.appendlist(key, field.value)
411+    return POST, FILES
412+
413+class StreamingUploadMiddleware:
414+
415+    def process_request(self, request):
416+        request.parse_file_upload = parse_streaming_file_upload
417+
418+def get_temp_file(identifier):
419+    return os.path.join(tempfile.gettempdir(),identifier)
420+
421+class UploadState:
422+
423+    def __init__(self, req):
424+        self.identifier = req.META['QUERY_STRING']
425+        self.state = {'size': int(req.header_dict.get('content-length')),
426+             'state': 'starting', 'received': 0}
427+        self.save()
428+
429+    def addlen(self, toadd):
430+        self.state['received'] = self.state['received'] + toadd
431+        if self.state['size']-1 <= self.state['received']:
432+            self.state['state'] = 'done'
433+        else:
434+             self.state['state'] = 'uploading'
435+        self.save()
436+
437+    def save(self):
438+        simplejson.dump(self.state,open(get_temp_file(self.identifier), 'w'))
439+
440+class UploadStateMiddleware:
441+    def process_request(self, request):
442+        if request.META['QUERY_STRING']:
443+            request.upload_state = UploadState
444+        if request.path == '/progress/':
445+            for header in request.header_dict.items():
446+                if header[0].upper().replace('-', '_').endswith('X_PROGRESS_ID'):
447+                    progress_id = header[1]
448+            try:
449+                content = open(get_temp_file(progress_id), 'r').read()
450+            except:
451+                content="{}"
452+            if not content:
453+                content="{}"
454+
455+            from django.http import HttpResponse
456+            return HttpResponse(content=content, mimetype='text/plain')