Ticket #4165: 5099-streaming_file_upload_with_safe_file_move_progress.diff

File 5099-streaming_file_upload_with_safe_file_move_progress.diff, 15.6 KB (added by Øyvind Saltvik <oyvind@…>, 8 years ago)

Start of new improved version

  • django/http/__init__.py

     
    1 import os
     1import os, pickle
    22from Cookie import SimpleCookie
    33from pprint import pformat
    44from urllib import urlencode, quote
    55from django.utils.datastructures import MultiValueDict
    66
     7try:
     8    from cStringIO import StringIO
     9except ImportError:
     10    from StringIO import StringIO
     11
    712RESERVED_CHARS="!*'();:@&=+$,/?%#[]"
    813
    914try:
     
    306311    if not host:
    307312        host = request.META.get('HTTP_HOST', '')
    308313    return host
     314
  • django/oldforms/__init__.py

     
    666666        self.validator_list = [self.isNonEmptyFile] + validator_list
    667667
    668668    def isNonEmptyFile(self, field_data, all_data):
    669         try:
    670             content = field_data['content']
    671         except TypeError:
     669        if field_data.has_key('_file_upload_error'):
     670            raise validators.CriticalValidationError, field_data['_file_upload_error']
     671        if not field_data.has_key('filename'):
    672672            raise validators.CriticalValidationError, gettext("No file was submitted. Check the encoding type on the form.")
    673         if not content:
     673        if not field_data['content-length']:
    674674            raise validators.CriticalValidationError, gettext("The submitted file is empty.")
    675675
    676676    def render(self, data):
    677677        return '<input type="file" id="%s" class="v%s" name="%s" />' % \
    678678            (self.get_id(), self.__class__.__name__, self.field_name)
    679679
     680    def prepare(self, new_data):
     681        if new_data.has_key('_file_upload_error'):
     682            # pretend we got something in the field to raise a validation error later
     683            new_data[self.field_name] = { '_file_upload_error': new_data['_file_upload_error'] }
     684
    680685    def html2python(data):
    681686        if data is None:
    682687            raise EmptyValue
  • django/db/models/base.py

     
    1212from django.dispatch import dispatcher
    1313from django.utils.datastructures import SortedDict
    1414from django.utils.functional import curry
     15from django.utils.file import file_move_safe
    1516from django.conf import settings
    1617from itertools import izip
    1718import types
    1819import sys
    1920import os
    2021
     22               
    2123class ModelBase(type):
    2224    "Metaclass for all models"
    2325    def __new__(cls, name, bases, attrs):
     
    361363    def _get_FIELD_size(self, field):
    362364        return os.path.getsize(self._get_FIELD_filename(field))
    363365
    364     def _save_FIELD_file(self, field, filename, raw_contents, save=True):
     366    def _save_FIELD_file(self, field, filename, raw_field, save=True):
    365367        directory = field.get_directory_name()
    366368        try: # Create the date-based directory if it doesn't exist.
    367369            os.makedirs(os.path.join(settings.MEDIA_ROOT, directory))
     
    383385        setattr(self, field.attname, filename)
    384386
    385387        full_filename = self._get_FIELD_filename(field)
    386         fp = open(full_filename, 'wb')
    387         fp.write(raw_contents)
    388         fp.close()
     388        if raw_field.has_key('tmpfilename'):
     389            raw_field['tmpfile'].close()
     390            file_move_safe(raw_field['tmpfilename'], full_filename)
     391        else:
     392            fp = open(full_filename, 'wb')
     393            fp.write(raw_field['content'])
     394            fp.close()
    389395
    390396        # Save the width and/or height, if applicable.
    391397        if isinstance(field, ImageField) and (field.width_field or field.height_field):
  • django/db/models/fields/__init__.py

     
    636636        setattr(cls, 'get_%s_filename' % self.name, curry(cls._get_FIELD_filename, field=self))
    637637        setattr(cls, 'get_%s_url' % self.name, curry(cls._get_FIELD_url, field=self))
    638638        setattr(cls, 'get_%s_size' % self.name, curry(cls._get_FIELD_size, field=self))
    639         setattr(cls, 'save_%s_file' % self.name, lambda instance, filename, raw_contents, save=True: instance._save_FIELD_file(self, filename, raw_contents, save))
     639        setattr(cls, 'save_%s_file' % self.name, lambda instance, filename, raw_field, save=True: instance._save_FIELD_file(self, filename, raw_field, save))
    640640        dispatcher.connect(self.delete_file, signal=signals.post_delete, sender=cls)
    641641
    642642    def delete_file(self, instance):
     
    659659        if new_data.get(upload_field_name, False):
    660660            func = getattr(new_object, 'save_%s_file' % self.name)
    661661            if rel:
    662                 func(new_data[upload_field_name][0]["filename"], new_data[upload_field_name][0]["content"], save)
     662                func(new_data[upload_field_name][0]["filename"], new_data[upload_field_name][0], save)
    663663            else:
    664                 func(new_data[upload_field_name]["filename"], new_data[upload_field_name]["content"], save)
     664                func(new_data[upload_field_name]["filename"], new_data[upload_field_name], save)
    665665
    666666    def get_directory_name(self):
    667667        return os.path.normpath(datetime.datetime.now().strftime(self.upload_to))
  • django/conf/global_settings.py

     
    240240# isExistingURL validator.
    241241URL_VALIDATOR_USER_AGENT = "Django/0.96pre (http://www.djangoproject.com)"
    242242
     243# The directory to place streamed file uploads. The web server needs write
     244# permissions on this directory.
     245# If this is None, streaming uploads are disabled.
     246FILE_UPLOAD_DIR = None
     247
     248
     249# The minimum size of a POST before file uploads are streamed to disk.
     250# Any less than this number, and the file is uploaded to memory.
     251# Size is in bytes.
     252STREAMING_MIN_POST_SIZE = 512 * (2**10)
     253
     254
     255
     256
    243257##############
    244258# MIDDLEWARE #
    245259##############
     
    335349
    336350# The list of directories to search for fixtures
    337351FIXTURE_DIRS = ()
     352
     353
  • django/core/handlers/wsgi.py

     
    111111            if self.environ.get('CONTENT_TYPE', '').startswith('multipart'):
    112112                header_dict = dict([(k, v) for k, v in self.environ.items() if k.startswith('HTTP_')])
    113113                header_dict['Content-Type'] = self.environ.get('CONTENT_TYPE', '')
    114                 self._post, self._files = http.parse_file_upload(header_dict, self.raw_post_data)
     114                header_dict['Content-Length'] = self.environ.get('CONTENT_LENGTH', '')
     115                try:
     116                    self._post, self._files = http.parse_file_upload(header_dict, self.environ['wsgi.input'], self.META.get('QUERY_STRING'))
     117                except:
     118                    self._post, self._files = {}, {} # make sure we dont read the input stream again
     119                    raise
     120                self._raw_post_data = None # raw data is not available for streamed multipart messages
    115121            else:
    116122                self._post, self._files = http.QueryDict(self.raw_post_data), datastructures.MultiValueDict()
    117123        else:
  • django/core/handlers/modpython.py

     
    4747    def _load_post_and_files(self):
    4848        "Populates self._post and self._files"
    4949        if 'content-type' in self._req.headers_in and self._req.headers_in['content-type'].startswith('multipart'):
    50             self._post, self._files = http.parse_file_upload(self._req.headers_in, self.raw_post_data)
     50            self._raw_post_data = None # raw data is not available for streamed multipart messages
     51            try:
     52                self._post, self._files = http.parse_file_upload(self._req.headers_in, self._req, self.META.get('QUERY_STRING'))
     53            except:
     54                self._post, self._files = {}, {} # make sure we dont read the input stream again
     55                raise
    5156        else:
    5257            self._post, self._files = http.QueryDict(self.raw_post_data), datastructures.MultiValueDict()
    5358
  • django/utils/file.py

     
     1import os
     2
     3try:
     4    import shutils
     5    file_move = shutils.move
     6except:
     7    file_move = os.rename
     8
     9def file_move_safe(old_file_name, new_file_name, chunk_size = 1024*64):
     10    """
     11    Moves a file from one location to another in the safest way possible.
     12   
     13    First, it tries using shutils.move, which is OS-dependent but doesn't
     14    break with change of filesystems. Then it tries os.rename, which will
     15    break if it encounters a change in filesystems. Lastly, it streams
     16    it manually from one file to another in python.
     17    """
     18   
     19    try:
     20        file_move(old_file_name, new_file_name)
     21        return
     22    except:
     23        pass
     24   
     25    new_file = open(new_file_name, 'wb')
     26    old_file = open(old_file_name, 'rb')
     27    current_chunk = None
     28   
     29    while current_chunk != '':
     30        current_chunk = old_file.read(chunk_size)
     31        new_file.write(current_chunk)
     32       
     33    new_file.close()
     34    old_file.close()
     35
     36    os.remove(old_file_name)
  • tests/modeltests/test_client/views.py

     
    4444
    4545    return HttpResponse(t.render(c))
    4646
     47def post_file_view(request):
     48    "A view that expects a multipart post and returns a file in the context"
     49    t = Template('File {{ file.filename }} received', name='POST Template')
     50    c = Context({'file': request.FILES['file_file']})
     51    return HttpResponse(t.render(c))
     52
    4753def redirect_view(request):
    4854    "A view that redirects all requests to the GET view"
    4955    return HttpResponseRedirect('/test_client/get_view/')
  • tests/modeltests/test_client/models.py

     
    7575        self.assertEqual(response.template.name, "Book template")
    7676        self.assertEqual(response.content, "Blink - Malcolm Gladwell")
    7777
     78    def test_post_file_view(self):
     79        "POST this python file to a view"
     80        import os, tempfile
     81        from django.conf import settings
     82        file = __file__.replace('.pyc', '.py')
     83        for upload_dir in [None, tempfile.gettempdir()]:
     84            settings.FILE_UPLOAD_DIR = upload_dir
     85            post_data = { 'name': file, 'file': open(file) }
     86            response = self.client.post('/test_client/post_file_view/', post_data)
     87            self.failUnless('models.py' in response.context['file']['filename'])
     88            self.failUnless(len(response.context['file']['content']) == os.path.getsize(file))
     89            if upload_dir:
     90                self.failUnless(response.context['file']['tmpfilename'])
     91
     92
    7893    def test_redirect(self):
    7994        "GET a URL that redirects elsewhere"
    8095        response = self.client.get('/test_client/redirect_view/')
  • tests/modeltests/test_client/urls.py

     
    44urlpatterns = patterns('',
    55    (r'^get_view/$', views.get_view),
    66    (r'^post_view/$', views.post_view),
     7    (r'^post_file_view/$', views.post_file_view),
    78    (r'^raw_post_view/$', views.raw_post_view),
    89    (r'^redirect_view/$', views.redirect_view),
    910    (r'^form_view/$', views.form_view),
  • docs/request_response.txt

     
    7272``FILES``
    7373    A dictionary-like object containing all uploaded files. Each key in
    7474    ``FILES`` is the ``name`` from the ``<input type="file" name="" />``. Each
    75     value in ``FILES`` is a standard Python dictionary with the following three
     75    value in ``FILES`` is a standard Python dictionary with the following four
    7676    keys:
    7777
    7878        * ``filename`` -- The name of the uploaded file, as a Python string.
    7979        * ``content-type`` -- The content type of the uploaded file.
    8080        * ``content`` -- The raw content of the uploaded file.
     81        * ``content-length`` -- The length of the content in bytes.
    8182
     83    If streaming file uploads are enabled two additional keys
     84    describing the uploaded file will be present:
     85
     86        * ``tmpfilename`` -- The filename for the temporary file.
     87        * ``tmpfile`` -- An open file object for the temporary file.
     88
     89    The temporary file will be removed when the request finishes.
     90
     91    Note that accessing ``content`` when streaming uploads are enabled
     92    will read the whole file into memory which may not be what you want.
     93
    8294    Note that ``FILES`` will only contain data if the request method was POST
    8395    and the ``<form>`` that posted to the request had
    8496    ``enctype="multipart/form-data"``. Otherwise, ``FILES`` will be a blank
  • docs/settings.txt

     
    437437
    438438.. _Testing Django Applications: ../testing/
    439439
     440FILE_UPLOAD_DIR
     441---------------
     442
     443Default: ``None``
     444
     445Path to a directory where temporary files should be written during
     446file uploads. Leaving this as ``None`` will disable streaming file uploads,
     447and cause all uploaded files to be stored (temporarily) in memory.
     448
    440449IGNORABLE_404_ENDS
    441450------------------
    442451
     
    780789
    781790.. _site framework docs: ../sites/
    782791
     792STREAMING_MIN_POST_SIZE
     793-----------------------
     794
     795Default: 524288 (``512*1024``)
     796
     797An integer specifying the minimum number of bytes that has to be
     798received (in a POST) for file upload streaming to take place. Any
     799request smaller than this will be handled in memory.
     800Note: ``FILE_UPLOAD_DIR`` has to be defined to enable streaming.
     801
    783802TEMPLATE_CONTEXT_PROCESSORS
    784803---------------------------
    785804
  • docs/forms.txt

     
    475475   new_data = request.POST.copy()
    476476   new_data.update(request.FILES)
    477477
     478Streaming file uploads.
     479-----------------------
     480
     481File uploads will be read into memory by default. This works fine for
     482small to medium sized uploads (from 1MB to 100MB depending on your
     483setup and usage). If you want to support larger uploads you can enable
     484upload streaming where only a small part of the file will be in memory
     485at any time. To do this you need to specify the ``FILE_UPLOAD_DIR``
     486setting (see the settings_ document for more details).
     487
     488See `request object`_ for more details about ``request.FILES`` objects
     489with streaming file uploads enabled.
     490
    478491Validators
    479492==========
    480493
     
    693706.. _`generic views`: ../generic_views/
    694707.. _`models API`: ../model-api/
    695708.. _settings: ../settings/
     709.. _request object: ../request_response/#httprequest-objects
Back to Top