Ticket #3119: store_filename_as_punycode_for_trunk_4462.diff

django/conf/global_settings.py

@@ -199 +199 @@
 # Example: "/home/media/media.lawrence.com/"
 MEDIA_ROOT = ''
 
+# If you set this to True, Django stores filename as punycode.
+STORE_FILENAME_AS_PUNYCODE = False
+
 # URL that handles the media served from MEDIA_ROOT.
 # Example: "http://media.lawrence.com"
 MEDIA_URL = ''
@@ -315 +318 @@
 # The name of the database to use for testing purposes.
 # If None, a name of 'test_' + DATABASE_NAME will be assumed
 TEST_DATABASE_NAME = None
+

django/db/models/fields/__init__.py

@@ -656 +656 @@
         return os.path.normpath(datetime.datetime.now().strftime(self.upload_to))
 
     def get_filename(self, filename):
-        from django.utils.text import get_valid_filename
+        if settings.STORE_FILENAME_AS_PUNYCODE :
+            from django.utils.text import get_valid_filename_as_puny as get_valid_filename
+        else :
+            from django.utils.text import get_valid_filename
         f = os.path.join(self.get_directory_name(), get_valid_filename(os.path.basename(filename)))
         return os.path.normpath(f)
 

django/utils/text.py

@@ +1 @@
+#encoding: utf-8
+
 import re
 
 from django.conf import settings
@@ -53 +55 @@
     s = s.strip().replace(' ', '_')
     return re.sub(r'[^-A-Za-z0-9_.]', '', s)
 
+def get_valid_filename_as_puny(s):
+    """
+    Encodes the given string in punycode except the extension.
+    Then generate a clean file name by calling get_valid_filename and return it.
+    >>> get_valid_filename_as_puny("développement image.jpg")
+    'dveloppement_image-kwa33c.jpg'
+    >>> get_valid_filename_as_puny("日本語ファイル名.jpg")
+    'caabc4cr5ao9bu0bzbf5exag15gcacc8bdc4c.jpg'
+    >>> get_valid_filename_as_puny("multidot.tar.gz")
+    'multidot.tar.gz'
+    >>> get_valid_filename_as_puny("developpement image.jpg")
+    'developpement_image.jpg'
+    >>> get_valid_filename_as_puny("john's portrait in 2004.jpg")
+    'johns_portrait_in_2004.jpg'
+    >>> get_valid_filename_as_puny("multidot.tar.gz")
+    'multidot.tar.gz'
+    >>> get_valid_filename_as_puny("複数ドット.tar.gz")
+    'daaaaqkd8g7f4e7ycc7a1a.tar.gz'
+    """
+    try:
+        dot_index = s.index('.')
+    except ValueError: # filename has no dot
+        filename = s.encode('punycode')
+        if len(filename) > 0 and filename[:-1] == s:
+            filenamme = s
+    else:
+        filename = s[:dot_index].encode('punycode') + s[dot_index:]
+        if len(filename[:filename.index('.')]) > 0 and filename[:filename.index('.') - 1] == s[:dot_index]:
+            filename = s
+    return get_valid_filename(filename)
+
 def get_text_list(list_, last_word='or'):
     """
     >>> get_text_list(['a', 'b', 'c', 'd'])

docs/settings.txt

@@ -711 +711 @@
 
 .. _site framework docs: ../sites/
 
+STORE_FILENAME_AS_PUNYCODE
+--------------------------
+
+Default: False
+
+Whether to encode filename with punycode. 
+By default, FileField and its subtype calls django.utils.text.get_valid_filename() to
+remove all "filename-unsafe" characters from given filename.
+The resulting filename consists of alphabets, numbers, hyphens and underscores.
+Set this True, if you might use multi-byte charcters for your filename.
+FileField encode their filename with punycode except extension.
+Then it remove all "filename-unsafe" characters.
+
+
 TEMPLATE_CONTEXT_PROCESSORS
 ---------------------------