Ticket #9764: idn_support_02.diff

django/db/models/fields/__init__.py

@@ -807 +807 @@
         return super(DecimalField, self).formfield(**defaults)
 
 class EmailField(CharField):
-    default_validators = [validators.validate_email]
+    default_validators = [validators.EmailValidator()]
     description = _("E-mail address")
 
     def __init__(self, *args, **kwargs):

django/forms/fields.py

@@ -413 +413 @@
     default_error_messages = {
         'invalid': _(u'Enter a valid e-mail address.'),
     }
-    default_validators = [validators.validate_email]
+    default_validators = [validators.EmailValidator()]
 
 class FileField(Field):
     widget = FileInput

django/core/validators.py

@@ -1 +1 @@
 import re
+import urlparse
 
 from django.core.exceptions import ValidationError
 from django.utils.translation import ugettext_lazy as _
@@ -52 +53 @@
         self.user_agent = validator_user_agent
 
     def __call__(self, value):
-        super(URLValidator, self).__call__(value)
+        try:
+            super(URLValidator, self).__call__(value)
+        except ValidationError, e:
+            # Trivial case failed. Try for possible IDN domain
+            if value:
+                original = value
+                value = smart_unicode(value)
+                splitted = urlparse.urlsplit(value)
+                try:
+                    netloc_ace = splitted[1].encode('idna') # IDN -> ACE
+                except UnicodeError: # invalid domain part
+                    raise e
+                value = value.replace(splitted[1], netloc_ace)
+                # If no URL path given, assume /
+                if not splitted[2]:
+                    value += u'/'
+                super(URLValidator, self).__call__(value)
+                # After validation revert ACE encoded domain-part to
+                # original (IDN) value as suggested by RFC 3490
+                value = original
+            else:
+                raise
+            
         if self.verify_exists:
             import urllib2
             headers = {
@@ -77 +100 @@
     except (ValueError, TypeError), e:
         raise ValidationError('')
 
+class EmailValidator(RegexValidator):
+    regex = re.compile(
+        r"(^[-!#$%&'*+/=?^_`{}|~0-9A-Z]+(\.[-!#$%&'*+/=?^_`{}|~0-9A-Z]+)*"  # dot-atom
+        r'|^"([\001-\010\013\014\016-\037!#-\[\]-\177]|\\[\001-011\013\014\016-\177])*"' # quoted-string
+        r')@(?:[A-Z0-9](?:[A-Z0-9-]{0,61}[A-Z0-9])?\.)+[A-Z]{2,6}\.?$', re.IGNORECASE)  # domain
+    message = _(u'Enter a valid e-mail address.')
 
-email_re = re.compile(
-    r"(^[-!#$%&'*+/=?^_`{}|~0-9A-Z]+(\.[-!#$%&'*+/=?^_`{}|~0-9A-Z]+)*"  # dot-atom
-    r'|^"([\001-\010\013\014\016-\037!#-\[\]-\177]|\\[\001-011\013\014\016-\177])*"' # quoted-string
-    r')@(?:[A-Z0-9](?:[A-Z0-9-]{0,61}[A-Z0-9])?\.)+[A-Z]{2,6}\.?$', re.IGNORECASE)  # domain
-validate_email = RegexValidator(email_re, _(u'Enter a valid e-mail address.'), 'invalid')
+    def __call__(self, value):
+        try:
+            super(EmailValidator, self).__call__(value)
+        except ValidationError, e:
+            # Trivial case failed. Try for possible IDN domain-part
+            if value and u'@' in value:
+                parts = value.split(u'@')
+                domain_part = parts[-1]
+                try:
+                    parts[-1] = parts[-1].encode('idna')
+                except UnicodeError:
+                    raise e
+                super(EmailValidator, self).__call__(u'@'.join(parts))
+            else:
+                raise
 
+validate_email = EmailValidator()
+
 slug_re = re.compile(r'^[-\w]+$')
 validate_slug = RegexValidator(slug_re, _(u"Enter a valid 'slug' consisting of letters, numbers, underscores or hyphens."), 'invalid')
 

tests/regressiontests/forms/fields.py

@@ -408 +408 @@
         self.assertRaisesErrorWithMessage(ValidationError, "[u'Enter a valid e-mail address.']", f.clean, 'example@inv-.-alid.com')
         self.assertEqual(u'example@valid-----hyphens.com', f.clean('example@valid-----hyphens.com'))
         self.assertEqual(u'example@valid-with-hyphens.com', f.clean('example@valid-with-hyphens.com'))
+        self.assertRaisesErrorWithMessage(ValidationError, "[u'Enter a valid e-mail address.']", f.clean, 'example@.com')
+        self.assertEqual(u'local@domain.with.idn.xyz\xe4\xf6\xfc\xdfabc.part.com', f.clean('local@domain.with.idn.xyzäöüßabc.part.com'))
 
     def test_email_regexp_for_performance(self):
         f = EmailField()
@@ -489 +491 @@
         self.assertRaisesErrorWithMessage(ValidationError, "[u'Enter a valid URL.']", f.clean, 'http://inv-.alid-.com')
         self.assertRaisesErrorWithMessage(ValidationError, "[u'Enter a valid URL.']", f.clean, 'http://inv-.-alid.com')
         self.assertEqual(u'http://valid-----hyphens.com/', f.clean('http://valid-----hyphens.com'))
+        self.assertEqual(u'http://some.idn.xyz\xe4\xf6\xfc\xdfabc.domain.com:123/blah', f.clean('http://some.idn.xyzäöüßabc.domain.com:123/blah'))
 
     def test_url_regex_ticket11198(self):
         f = URLField()

docs/ref/forms/fields.txt

@@ -482 +482 @@
 If provided, these arguments ensure that the string is at most or at least the
 given length.
 
+.. versionchanged:: 1.2
+   The EmailField previously did not recognize e-mail addresses as valid that
+   contained an IDN (Internationalized Domain Name; a domain containing 
+   unicode characters) domain part. This has now been corrected.
+
 ``FileField``
 ~~~~~~~~~~~~~
 
@@ -707 +712 @@
     String used as the user-agent used when checking for a URL's existence.
     Defaults to the value of the ``URL_VALIDATOR_USER_AGENT`` setting.
 
+.. versionchanged:: 1.2
+   The URLField previously did not recognize URLs as valid that contained an IDN
+   (Internationalized Domain Name; a domain name containing unicode characters)
+   domain name. This has now been corrected.
+
+
 Slightly complex built-in ``Field`` classes
 -------------------------------------------
 

docs/ref/forms/validation.txt

@@ -186 +186 @@
         default_error_messages = {
             'invalid': _(u'Enter a valid e-mail address.'),
         }
-        default_validators = [validators.validate_email]
+        default_validators = [validators.EMailValidator()]
 
 As you can see, ``EmailField`` is just a ``CharField`` with customized error
 message and a validator that validates e-mail addresses. This can also be done
@@ -196 +196 @@
 
 is equivalent to::
 
-    email = forms.CharField(validators=[validators.validate_email],
+    email = forms.CharField(validators=[validators.EMailValidator()],
             error_messages={'invalid': _(u'Enter a valid e-mail address.')})