Ticket #2588: r3645_unicode.patch

django/http/__init__.py

@@ -172 +172 @@
 
     def __str__(self):
         "Full HTTP message, including headers"
+        # Coerce the output to be in the correct charset
+        # Note that we are doing this at the last possible moment
+        # because we assume that all strings internal in django 
+        # are proper unicode strings, or they're the ASCII portion
+        # of UTF-8 (codepoint 127 or less)
         return '\n'.join(['%s: %s' % (key, value)
             for key, value in self.headers.items()]) \
-            + '\n\n' + self.content
+            + '\n\n' + self.content.encode(settings.DEFAULT_CHARSET)
 
     def __setitem__(self, header, value):
         self.headers[header] = value

django/db/models/fields/__init__.py

@@ -290 +290 @@
             return first_choice + list(self.choices)
         rel_model = self.rel.to
         if hasattr(self.rel, 'get_related_field'):
-            lst = [(getattr(x, self.rel.get_related_field().attname), str(x)) for x in rel_model._default_manager.complex_filter(self.rel.limit_choices_to)]
+            lst = [(getattr(x, self.rel.get_related_field().attname), unicode(x)) for x in rel_model._default_manager.complex_filter(self.rel.limit_choices_to)]
         else:
-            lst = [(x._get_pk_val(), str(x)) for x in rel_model._default_manager.complex_filter(self.rel.limit_choices_to)]
+            lst = [(x._get_pk_val(), unicode(x)) for x in rel_model._default_manager.complex_filter(self.rel.limit_choices_to)]
         return first_choice + lst
 
     def get_choices_default(self):

django/forms/__init__.py

@@ -158 +158 @@
 
     def __str__(self):
         "Renders the field"
-        return str(self.formfield.render(self.data))
+        return unicode(self.formfield.render(self.data))
 
     def __repr__(self):
         return '<FormFieldWrapper for "%s">' % self.formfield.field_name
@@ -392 +392 @@
         if self.maxlength:
             maxlength = 'maxlength="%s" ' % self.maxlength
         if isinstance(data, unicode):
-            data = data.encode(settings.DEFAULT_CHARSET)
+            #data = data.encode(settings.DEFAULT_CHARSET)
+            data = unicode(data)
         return '<input type="%s" id="%s" class="v%s%s" name="%s" size="%s" value="%s" %s/>' % \
             (self.input_type, self.get_id(), self.__class__.__name__, self.is_required and ' required' or '',
             self.field_name, self.length, escape(data), maxlength)
@@ -418 +419 @@
         if data is None:
             data = ''
         if isinstance(data, unicode):
-            data = data.encode(settings.DEFAULT_CHARSET)
+            #data = data.encode(settings.DEFAULT_CHARSET)
+            data = unicode(data)
         return '<textarea id="%s" class="v%s%s" name="%s" rows="%s" cols="%s">%s</textarea>' % \
             (self.get_id(), self.__class__.__name__, self.is_required and ' required' or '',
             self.field_name, self.rows, self.cols, escape(data))

django/core/urlresolvers.py

@@ -79 +79 @@
             test_regex = grouped
         # Note we're using re.match here on purpose because the start of
         # to string needs to match.
-        if not re.match(test_regex + '$', str(value)): # TODO: Unicode?
+        if not re.match(test_regex + '$', unicode(value), re.UNICODE):
             raise NoReverseMatch("Value %r didn't match regular expression %r" % (value, test_regex))
-        return str(value) # TODO: Unicode?
+        return unicode(value)
 
 class RegexURLPattern(object):
     def __init__(self, regex, callback, default_args=None):
@@ -89 +89 @@
         # callback is either a string like 'foo.views.news.stories.story_detail'
         # which represents the path to a module and a view function name, or a
         # callable object (view).
-        self.regex = re.compile(regex)
+        self.regex = re.compile(regex, re.UNICODE)
         if callable(callback):
             self._callback = callback
         else:
@@ -143 +143 @@
     def __init__(self, regex, urlconf_name, default_kwargs=None):
         # regex is a string representing a regular expression.
         # urlconf_name is a string representing the module containing urlconfs.
-        self.regex = re.compile(regex)
+        self.regex = re.compile(regex, re.UNICODE)
         self.urlconf_name = urlconf_name
         self.callback = None
         self.default_kwargs = default_kwargs or {}
@@ -225 +225 @@
         from django.conf import settings
         urlconf = settings.ROOT_URLCONF
     resolver = RegexURLResolver(r'^/', urlconf)
+
     return resolver.resolve(path)
 
 def reverse(viewname, urlconf=None, args=None, kwargs=None):

django/core/handlers/base.py

@@ -50 +50 @@
 
     def get_response(self, path, request):
         "Returns an HttpResponse object for the given HttpRequest"
+        from exceptions import UnicodeDecodeError
         from django.core import exceptions, urlresolvers
         from django.core.mail import mail_admins
         from django.conf import settings
 
+        # URLs are encoded in UTF8 (RFC3986, RFC2718)
+
+        try:
+            path = path.decode('utf8')
+        except UnicodeDecodeError, ude:
+            raise exceptions.SuspiciousOperation, "Failed to decode the URL using UTF8.  This is probably a user agent."
+
         # Apply request middleware
         for middleware_method in self._request_middleware:
             response = middleware_method(request)

django/template/__init__.py

@@ -688 +688 @@
                 bits.append(self.render_node(node, context))
             else:
                 bits.append(node)
-        return ''.join(bits)
+        return u''.join(bits)
 
     def get_nodes_by_type(self, nodetype):
         "Return a list of all nodes of the given type"
@@ -734 +734 @@
         return "<Variable Node: %s>" % self.filter_expression
 
     def encode_output(self, output):
-        # Check type so that we don't run str() on a Unicode object
+        # We only want unicode.  encoding to the default charset (utf8 for
+        # monkeybean) will result in a string of indeterminate encoding.
+        # We want to just convert the whole damn document into the right
+        # encoding just before we write to the output stream.  That happens in
+        # django.http.__init__
+        return unicode(output)
+
+        '''
         if not isinstance(output, basestring):
-            return str(output)
+            return unicode(output)
         elif isinstance(output, unicode):
             return output.encode(settings.DEFAULT_CHARSET)
         else:
-            return output
+            return unicode(output)
+        '''
 
     def render(self, context):
         output = self.filter_expression.resolve(context)

tests/othertests/urlpatterns_reverse.py

@@ -23 +23 @@
     ('^people/(?P<state>\w\w)/(?P<name>\w+)/$', NoReverseMatch, [], {'name': 'adrian'}),
     ('^people/(?P<state>\w\w)/(\w+)/$', NoReverseMatch, ['il'], {'name': 'adrian'}),
     ('^people/(?P<state>\w\w)/(\w+)/$', 'people/il/adrian/', ['adrian'], {'state': 'il'}),
+    ('^people/(?P<state>\w\w)/(\w+)/$', 'people/il/adrian/', ['adrian'], {'state': 'il'}),
+    ('^people/(?P<state>\w\w)/(?P<name>\w+)/$', u'people/il/adria\xd0/', [], {'state': u'il', 'name': u'adria\xd0'}),
 )
 
 def run_tests(verbosity=0):