Ticket #16052: mysql_utf8bin_unicode.diff

File mysql_utf8bin_unicode.diff, 4.5 KB (added by graham_king, 4 years ago)

Tries to guess whether a VAR_STRING has binary or unicode data, and converts appropriately.

  • tests/regressiontests/backends/tests.py

     
    11# -*- coding: utf-8 -*-
    22# Unit and doctests for specific database backends.
    33import datetime
     4import sys
    45
    56from django.conf import settings
    67from django.core.management.color import no_style
     
    322323            a.save()
    323324        except IntegrityError:
    324325            pass
     326
     327class MySQLEncodingTest(TestCase):
     328
     329    @unittest.skipUnless(connection.vendor == 'mysql',
     330                         "MySQL specific utf8_bin test.")
     331    def test_utfbin_unicode(self):
     332        """
     333        Ensure CharField is unicode with both utf_bin and default encoding.
     334        """
     335
     336        models.Person.objects.create(first_name='Django', last_name='Reinhardt')
     337
     338        cur = connections['default'].cursor()
     339        cur.execute('ALTER TABLE backends_person ' +
     340                    'MODIFY first_name VARCHAR(20) ' +
     341                    'CHARACTER SET utf8 COLLATE utf8_bin;')
     342        cur.close()
     343
     344        person = models.Person.objects.get(last_name='Reinhardt')
     345
     346        self.assertIsInstance(person.first_name, unicode)   # utf_bin
     347        self.assertIsInstance(person.last_name, unicode)    # default
     348
     349    @unittest.skipUnless(connection.vendor == 'mysql',
     350                         "MySQL specific utf8_bin test.")
     351    def test_utfbin_unicode_needed(self):
     352        """
     353        Ensure CharField is unicode with both utf_bin and default encoding,
     354        with an actual non-ascii string.
     355        """
     356
     357        models.Person.objects.create(first_name='汉语/漢語', last_name='Reinhardt')
     358
     359        cur = connections['default'].cursor()
     360        cur.execute('ALTER TABLE backends_person ' +
     361                    'MODIFY first_name VARCHAR(20) ' +
     362                    'CHARACTER SET utf8 COLLATE utf8_bin;')
     363        cur.close()
     364
     365        person = models.Person.objects.get(last_name='Reinhardt')
     366
     367        self.assertIsInstance(person.first_name, unicode)   # utf_bin
     368        self.assertIsInstance(person.last_name, unicode)    # default
     369
     370    @unittest.skipUnless(connection.vendor == 'mysql',
     371                         "MySQL specific conversion test.")
     372    def test_convert_var_string_unicode(self):
     373        """Test the convert_var_string method with unicode data"""
     374        from django.db.backends.mysql.base import convert_var_string
     375
     376        result = convert_var_string('汉语/漢語')
     377        self.assertIsInstance(result, unicode)
     378
     379    @unittest.skipUnless(connection.vendor == 'mysql',
     380                         "MySQL specific conversion test.")
     381    def test_convert_var_string_binary(self):
     382        """Test the convert_var_string method with binary data"""
     383        from django.db.backends.mysql.base import convert_var_string
     384
     385        # Use the python executable as source of binary data
     386        data = open(sys.executable, 'rb').read()
     387
     388        result = convert_var_string(data)
     389        self.assertIsInstance(result, str)
     390
  • django/db/backends/mysql/base.py

     
    3333from django.db.backends.mysql.introspection import DatabaseIntrospection
    3434from django.db.backends.mysql.validation import DatabaseValidation
    3535from django.utils.safestring import SafeString, SafeUnicode
     36from django.utils.encoding import force_unicode
    3637
    3738# Raise exceptions for database warnings if DEBUG is on
    3839from django.conf import settings
     
    4344DatabaseError = Database.DatabaseError
    4445IntegrityError = Database.IntegrityError
    4546
     47def convert_var_string(data):
     48    """
     49    Guess whether data is binary or text, and return:
     50     - str for binary and UTF-16
     51     - unicode for UTF-8, ascii and all other encodings.
     52    """
     53    if '\0' in data:
     54        return str(data)
     55    else:
     56        return force_unicode(data)
     57
    4658# MySQLdb-1.2.1 returns TIME columns as timedelta -- they are more like
    4759# timedelta in terms of actual behavior as they are signed and include days --
    4860# and Django expects time, so we still need to override that. We also need to
     
    5365    FIELD_TYPE.TIME: util.typecast_time,
    5466    FIELD_TYPE.DECIMAL: util.typecast_decimal,
    5567    FIELD_TYPE.NEWDECIMAL: util.typecast_decimal,
     68    FIELD_TYPE.VAR_STRING: [(None, convert_var_string)],
    5669})
    5770
    5871# This should match the numerical portion of the version numbers (we can treat
Back to Top