Code

Ticket #4165: 5099_upload_middleware_and_oldadmin_js.diff

File 5099_upload_middleware_and_oldadmin_js.diff, 20.9 KB (added by Michael Axiak <axiak@…>, 7 years ago)

The Middleware and JavaScript

Line 
1Index: django/contrib/admin/media/js/UploadProgress.js
2===================================================================
3--- django/contrib/admin/media/js/UploadProgress.js     (revision 0)
4+++ django/contrib/admin/media/js/UploadProgress.js     (revision 0)
5@@ -0,0 +1,103 @@
6+
7+function getxy(){
8+    var x,y;
9+    if (self.innerHeight) // all except Explorer
10+        {
11+        x = self.innerWidth;
12+        y = self.innerHeight;
13+        }
14+    else if (document.documentElement && document.documentElement.clientHeight)
15+        // Explorer 6 Strict Mode
16+        {
17+        x = document.documentElement.clientWidth;
18+        y = document.documentElement.clientHeight;
19+        }
20+    else if (document.body) // other Explorers
21+        {
22+        x = document.body.clientWidth;
23+        y = document.body.clientHeight;
24+        }
25+    return {'x':x,'y':y}
26+    }
27+
28+var humanvalue = ['B','KB','MB','GB']
29+function humanize(bytes) {
30+    curbytes = bytes
31+    iterations = 0
32+    while (curbytes>1024) {
33+        iterations++
34+        curbytes=curbytes/1024
35+        }
36+    return curbytes.toFixed(1) + ' ' + humanvalue[iterations]
37+    }
38+
39+interval = null;
40+function fetch(uuid) {
41+    req = xmlhttp
42+    req.open("GET", "/progress/", 1);
43+    req.setRequestHeader("X-Progress-Id", uuid);
44+    req.onreadystatechange = function () {
45+    if (req.readyState == 4) {
46+        if (req.status == 200) {
47+
48+            var upload = eval( '(' + req.responseText + ')' );
49+
50+            if (upload.state == 'done' || upload.state == 'uploading') {
51+                bar = document.getElementById('progress_bar');
52+                bar_txt = document.getElementById('progress_text')
53+                bar_txt.innerHTML = ((upload.received / upload.size) * 100).toFixed(1) + '% - ' +
54+                    humanize(upload.received) + ' of ' + humanize(upload.size)
55+                w = 400 * upload.received / upload.size;
56+                bar.style.width = w + 'px';
57+
58+                }
59+                if (upload.state == 'done') {
60+                    window.clearTimeout(interval);
61+                    }
62+                }
63+            }
64+        }
65+    req.send(null);
66+
67+    }
68+
69+function openprogress(e) {
70+
71+    uuid = "";
72+    for (i = 0; i < 32; i++) {
73+        uuid += Math.floor(Math.random() * 16).toString(16);
74+        }
75+    frm = e.target||e.srcElement
76+
77+    frm.action=frm.action+"?" + uuid;
78+
79+    pos = getxy()
80+    posx = parseInt((pos.x/2)-(420/2), 10)
81+    posy = parseInt((pos.y/2)-(50/2), 10)
82+
83+    progress_wrap = quickElement('div', document.body, '', 'style',
84+        'position: absolute; top: '+posy+'px; left: '+posx+'px; height: 50px; ' +
85+        'padding: 10px; width: 420px; background: #ffffff; ' +
86+        'border: solid 1px #dddddd;', 'id', 'progress_wrap')
87+
88+    progress_label = quickElement('h1', progress_wrap, 'Upload progress')
89+
90+    progress = quickElement('div', progress_wrap, '', 'style',
91+        'top: 0; left: 0; width: 0px; ', 'id', 'progress_bar', 'class', 'submit-row')
92+
93+    progress_text = quickElement('div', progress_wrap, '0%', 'style',
94+        'color: #000000; ', 'id', 'progress_text')
95+
96+    interval = window.setInterval(
97+        function () {
98+            fetch(uuid);
99+            },
100+        1000
101+        );
102+    }
103+
104+addEvent(window, 'load', function() {
105+        frm = document.getElementsByTagName('form')[0]
106+        addEvent(frm, 'submit',  openprogress)   
107+        }
108+    )
109Index: django/middleware/upload.py
110===================================================================
111--- django/middleware/upload.py (revision 0)
112+++ django/middleware/upload.py (revision 0)
113@@ -0,0 +1,416 @@
114+"streaming upload middleware"
115+import cgi
116+import os
117+import tempfile
118+import re
119+import email, email.Message, email.FeedParser
120+from email import Errors
121+from email import Message
122+from email.FeedParser import NLCRE, NLCRE_bol, NLCRE_eol, NLCRE_crack,headerRE, EMPTYSTRING, NL, NeedMoreData
123+
124+from django.conf import settings
125+from django.utils.datastructures import MultiValueDict
126+from django.utils import simplejson
127+
128+from django.core import signals
129+from django.dispatch import dispatcher
130+
131+def delete_tempfile(sender):
132+    sender.delete()
133+
134+class TempFileDict(dict):
135+    "Keeps uploaded file as a file-like object and reads its content on demand"
136+   
137+    def __init__(self):
138+        self['name'] = tempfile.mktemp()
139+        self['file'] = open(self['name'], "w+b")
140+        dispatcher.connect(delete_tempfile, sender=self, signal=signals.request_finished)
141+
142+    def write(self, data):
143+        self['file'].write(data)
144+
145+    def close(self):
146+        self['file'].close()
147+
148+    def delete(self):
149+        if os.path.exsists(self['name']):
150+            os.remove(self['name'])
151+   
152+    def tell(self):
153+        return os.path.getsize(self['name'])
154+
155+    def __repr__(self):
156+        return '<TempFileDict>'
157+
158+class FileDict(dict):
159+
160+    def __init__(self, filename, contenttype, payload):
161+        self['filename'] , self['content-type'], self['payload'] = filename, contenttype, payload
162+
163+    def __getitem__(self, name):
164+        if name=='content' and not 'content' in self:
165+            size = self.tell()
166+            f = open(self['payload']['name'], 'r')
167+            self['content'] = f.read(size)
168+            f.close()
169+        return dict.__getitem__(self, name)
170+
171+    def tell(self):
172+        return self['payload'].tell()
173+
174+    def __repr__(self):
175+        return '<FileDict>'
176+
177+
178+class StreamingFileFeedParser(email.FeedParser.FeedParser):
179+
180+    def _parsegen(self):
181+        # Create a new message and start by parsing headers.
182+        self._new_message()
183+        headers = []
184+        # Collect the headers, searching for a line that doesn't match the RFC
185+        # 2822 header or continuation pattern (including an empty line).
186+        for line in self._input:
187+            if line is NeedMoreData:
188+                yield NeedMoreData
189+                continue
190+            if not headerRE.match(line):
191+                # If we saw the RFC defined header/body separator
192+                # (i.e. newline), just throw it away. Otherwise the line is
193+                # part of the body so push it back.
194+                if not NLCRE.match(line):
195+                    self._input.unreadline(line)
196+                break
197+            headers.append(line)
198+        # Done with the headers, so parse them and figure out what we're
199+        # supposed to see in the body of the message.
200+        self._parse_headers(headers)
201+        # Headers-only parsing is a backwards compatibility hack, which was
202+        # necessary in the older parser, which could throw errors.  All
203+        # remaining lines in the input are thrown into the message body.
204+        if self._headersonly:
205+            lines = []
206+            while True:
207+                line = self._input.readline()
208+                if line is NeedMoreData:
209+                    yield NeedMoreData
210+                    continue
211+                if line == '':
212+                    break
213+                lines.append(line)
214+            self._cur.set_payload(EMPTYSTRING.join(lines))
215+            return
216+        if self._cur.get_content_type() == 'message/delivery-status':
217+            # message/delivery-status contains blocks of headers separated by
218+            # a blank line.  We'll represent each header block as a separate
219+            # nested message object, but the processing is a bit different
220+            # than standard message/* types because there is no body for the
221+            # nested messages.  A blank line separates the subparts.
222+            while True:
223+                self._input.push_eof_matcher(NLCRE.match)
224+                for retval in self._parsegen():
225+                    if retval is NeedMoreData:
226+                        yield NeedMoreData
227+                        continue
228+                    break
229+                msg = self._pop_message()
230+                # We need to pop the EOF matcher in order to tell if we're at
231+                # the end of the current file, not the end of the last block
232+                # of message headers.
233+                self._input.pop_eof_matcher()
234+                # The input stream must be sitting at the newline or at the
235+                # EOF.  We want to see if we're at the end of this subpart, so
236+                # first consume the blank line, then test the next line to see
237+                # if we're at this subpart's EOF.
238+                while True:
239+                    line = self._input.readline()
240+                    if line is NeedMoreData:
241+                        yield NeedMoreData
242+                        continue
243+                    break
244+                while True:
245+                    line = self._input.readline()
246+                    if line is NeedMoreData:
247+                        yield NeedMoreData
248+                        continue
249+                    break
250+                if line == '':
251+                    break
252+                # Not at EOF so this is a line we're going to need.
253+                self._input.unreadline(line)
254+            return
255+        if self._cur.get_content_maintype() == 'message':
256+            # The message claims to be a message/* type, then what follows is
257+            # another RFC 2822 message.
258+            for retval in self._parsegen():
259+                if retval is NeedMoreData:
260+                    yield NeedMoreData
261+                    continue
262+                break
263+            self._pop_message()
264+            return
265+        if self._cur.get_content_maintype() == 'multipart':
266+            boundary = self._cur.get_boundary()
267+            if boundary is None:
268+                # The message /claims/ to be a multipart but it has not
269+                # defined a boundary.  That's a problem which we'll handle by
270+                # reading everything until the EOF and marking the message as
271+                # defective.
272+                self._cur.defects.append(Errors.NoBoundaryInMultipartDefect())
273+                lines = []
274+                for line in self._input:
275+                    if line is NeedMoreData:
276+                        yield NeedMoreData
277+                        continue
278+                    lines.append(line)
279+                self._cur.set_payload(EMPTYSTRING.join(lines))
280+                return
281+            # Create a line match predicate which matches the inter-part
282+            # boundary as well as the end-of-multipart boundary.  Don't push
283+            # this onto the input stream until we've scanned past the
284+            # preamble.
285+            separator = '--' + boundary
286+            boundaryre = re.compile(
287+                '(?P<sep>' + re.escape(separator) +
288+                r')(?P<end>--)?(?P<ws>[ \t]*)(?P<linesep>\r\n|\r|\n)?$')
289+            capturing_preamble = True
290+            preamble = []
291+            linesep = False
292+            while True:
293+                line = self._input.readline()
294+                if line is NeedMoreData:
295+                    yield NeedMoreData
296+                    continue
297+                if line == '':
298+                    break
299+                mo = boundaryre.match(line)
300+                if mo:
301+                    # If we're looking at the end boundary, we're done with
302+                    # this multipart.  If there was a newline at the end of
303+                    # the closing boundary, then we need to initialize the
304+                    # epilogue with the empty string (see below).
305+                    if mo.group('end'):
306+                        linesep = mo.group('linesep')
307+                        break
308+                    # We saw an inter-part boundary.  Were we in the preamble?
309+                    if capturing_preamble:
310+                        if preamble:
311+                            # According to RFC 2046, the last newline belongs
312+                            # to the boundary.
313+                            lastline = preamble[-1]
314+                            eolmo = NLCRE_eol.search(lastline)
315+                            if eolmo:
316+                                preamble[-1] = lastline[:-len(eolmo.group(0))]
317+                            self._cur.preamble = EMPTYSTRING.join(preamble)
318+                        capturing_preamble = False
319+                        self._input.unreadline(line)
320+                        continue
321+                    # We saw a boundary separating two parts.  Consume any
322+                    # multiple boundary lines that may be following.  Our
323+                    # interpretation of RFC 2046 BNF grammar does not produce
324+                    # body parts within such double boundaries.
325+                    while True:
326+                        line = self._input.readline()
327+                        if line is NeedMoreData:
328+                            yield NeedMoreData
329+                            continue
330+                        mo = boundaryre.match(line)
331+                        if not mo:
332+                            self._input.unreadline(line)
333+                            break
334+                    # Recurse to parse this subpart; the input stream points
335+                    # at the subpart's first line.
336+                    self._input.push_eof_matcher(boundaryre.match)
337+                    for retval in self._parsegen():
338+                        if retval is NeedMoreData:
339+                            yield NeedMoreData
340+                            continue
341+                        break
342+                    # Because of RFC 2046, the newline preceding the boundary
343+                    # separator actually belongs to the boundary, not the
344+                    # previous subpart's payload (or epilogue if the previous
345+                    # part is a multipart).
346+                    if self._last.get_content_maintype() == 'multipart':
347+                        epilogue = self._last.epilogue
348+                        if epilogue == '':
349+                            self._last.epilogue = None
350+                        elif epilogue is not None:
351+                            mo = NLCRE_eol.search(epilogue)
352+                            if mo:
353+                                end = len(mo.group(0))
354+                                self._last.epilogue = epilogue[:-end]
355+                    else:
356+                        payload = self._last.get_payload()
357+                        if isinstance(payload, basestring):
358+                            mo = NLCRE_eol.search(payload)
359+                            if mo:
360+                                payload = payload[:-len(mo.group(0))]
361+                                self._last.set_payload(payload)
362+                    self._input.pop_eof_matcher()
363+                    self._pop_message()
364+                    # Set the multipart up for newline cleansing, which will
365+                    # happen if we're in a nested multipart.
366+                    self._last = self._cur
367+                else:
368+                    # I think we must be in the preamble
369+                    assert capturing_preamble
370+                    preamble.append(line)
371+            # We've seen either the EOF or the end boundary.  If we're still
372+            # capturing the preamble, we never saw the start boundary.  Note
373+            # that as a defect and store the captured text as the payload.
374+            # Everything from here to the EOF is epilogue.
375+            if capturing_preamble:
376+                self._cur.defects.append(Errors.StartBoundaryNotFoundDefect())
377+                self._cur.set_payload(EMPTYSTRING.join(preamble))
378+                epilogue = []
379+                for line in self._input:
380+                    if line is NeedMoreData:
381+                        yield NeedMoreData
382+                        continue
383+                self._cur.epilogue = EMPTYSTRING.join(epilogue)
384+                return
385+            # If the end boundary ended in a newline, we'll need to make sure
386+            # the epilogue isn't None
387+            if linesep:
388+                epilogue = ['']
389+            else:
390+                epilogue = []
391+            for line in self._input:
392+                if line is NeedMoreData:
393+                    yield NeedMoreData
394+                    continue
395+                epilogue.append(line)
396+            # Any CRLF at the front of the epilogue is not technically part of
397+            # the epilogue.  Also, watch out for an empty string epilogue,
398+            # which means a single newline.
399+            if epilogue:
400+                firstline = epilogue[0]
401+                bolmo = NLCRE_bol.match(firstline)
402+                if bolmo:
403+                    epilogue[0] = firstline[len(bolmo.group(0)):]
404+            self._cur.epilogue = EMPTYSTRING.join(epilogue)
405+            return
406+        # Otherwise, it's some non-multipart type, so the entire rest of the
407+        # file contents becomes the payload.
408+        name_dict = cgi.parse_header(self._cur['Content-Disposition'])[1]
409+        if name_dict.has_key('filename'):
410+            tmpfile = TempFileDict()
411+            for line in self._input:
412+                if line is NeedMoreData:
413+                    yield NeedMoreData
414+                    continue
415+                tmpfile.write(line)
416+            tmpfile.close()
417+            self._cur.set_payload(tmpfile)
418+        else:
419+            lines = []
420+            for line in self._input:
421+                if line is NeedMoreData:
422+                    yield NeedMoreData
423+                    continue
424+                lines.append(line)
425+            self._cur.set_payload(EMPTYSTRING.join(lines))
426+
427+def parse_streaming_file_upload(req):
428+    "Returns a tuple of (POST MultiValueDict, FILES MultiValueDict)"
429+
430+    try:
431+        BUFFER_SIZE=settings.UPLOAD_BUFFER_SIZE
432+    except:
433+        BUFFER_SIZE=200000
434+
435+    if hasattr(req, 'upload_state'):
436+        upload_state = req.upload_state(req)
437+    else:
438+        upload_state = None
439+
440+    raw_headers = '\r\n'.join(['%s:%s' % pair for pair in req.header_dict.items()])
441+    raw_headers += '\r\n\r\n'
442+    POST = MultiValueDict()
443+    FILES = MultiValueDict()
444+    parser = StreamingFileFeedParser()
445+    parser.feed(raw_headers)
446+    while 1:
447+        # make this a non-blocing read
448+        line=req.raw_request.read(BUFFER_SIZE)
449+        if upload_state:
450+            upload_state.addlen(len(line))
451+        parser.feed(line)
452+        if line == '':
453+            break
454+    msg=parser.close()
455+    POST = MultiValueDict()
456+    FILES = MultiValueDict()
457+    for submessage in msg.get_payload():
458+        if isinstance(submessage, email.Message.Message):
459+            name_dict = cgi.parse_header(submessage['Content-Disposition'])[1]
460+            # name_dict is something like {'name': 'file', 'filename': 'test.txt'} for file uploads
461+            # or {'name': 'blah'} for POST fields
462+            # We assume all uploaded files have a 'filename' set.
463+            if name_dict.has_key('filename'):
464+                assert type([]) != type(submessage.get_payload()), "Nested MIME messages are not supported"
465+                if not name_dict['filename'].strip():
466+                    continue
467+                # IE submits the full path, so trim everything but the basename.
468+                # (We can't use os.path.basename because it expects Linux paths.)
469+                filename = name_dict['filename'][name_dict['filename'].rfind("\\")+1:]
470+                FILES.appendlist(name_dict['name'], FileDict(
471+                    filename,
472+                    (submessage.has_key('Content-Type') and submessage['Content-Type'] or None),
473+                    submessage.get_payload()
474+                ))
475+            else:
476+                POST.appendlist(name_dict['name'], submessage.get_payload())
477+    return POST, FILES
478+
479+class StreamingUploadMiddleware:
480+
481+    def process_request(self, request):
482+        request.parse_file_upload = parse_streaming_file_upload
483+
484+def get_temp_file(identifier):
485+    return os.path.join(tempfile.gettempdir(),identifier)
486+
487+class UploadState:
488+
489+    def __init__(self, req):
490+        self.identifier = req.META['QUERY_STRING']
491+        self.state = {'size': int(req.header_dict.get('content-length')),
492+             'state': 'starting', 'received': 0}
493+        self.save()
494+
495+    def addlen(self, toadd):
496+        self.state['received'] = self.state['received'] + toadd
497+        if self.state['size']-1 <= self.state['received']:
498+            self.state['state'] = 'done'
499+        else:
500+             self.state['state'] = 'uploading'
501+        self.save()
502+
503+    def save(self):
504+        simplejson.dump(self.state,open(get_temp_file(self.identifier), 'w'))
505+
506+class UploadStateMiddleware:
507+
508+    def process_request(self, request):
509+
510+        try:
511+            progress_url=settings.PROGRESS_URL
512+        except:
513+            progress_url='/progress/'
514+
515+        if request.META['QUERY_STRING']:
516+            request.upload_state = UploadState
517+
518+        if request.path == progress_url:
519+            progress_id = request.progress_id
520+
521+            try:
522+                content = open(get_temp_file(progress_id), 'r').read()
523+            except:
524+                content="{}"
525+            if not content:
526+                content="{}"
527+
528+            from django.http import HttpResponse
529+            return HttpResponse(content=content, mimetype='text/plain')