Ticket #6271: smart_split.2.diff

django/utils/text.py

@@ -178 +178 @@
 ustring_re = re.compile(u"([\u0080-\uffff])")
 
 def javascript_quote(s, quote_double_quotes=False):
-
     def fix(match):
         return r"\u%04x" % ord(match.group(1))
 
@@ -196 +195 @@
     return str(ustring_re.sub(fix, s))
 javascript_quote = allow_lazy(javascript_quote, unicode)
 
-smart_split_re = re.compile('("(?:[^"\\\\]*(?:\\\\.[^"\\\\]*)*)"|\'(?:[^\'\\\\]*(?:\\\\.[^\'\\\\]*)*)\'|[^\\s]+)')
+smart_split_re = re.compile(r'("(?:[^"\\]*(?:\\.[^"\\]*)*)"'
+                            r"|'(?:[^'\\]*(?:\\.[^'\\]*)*)'"
+                            r'|[^\s"\']+|["\'])|\s+')
 def smart_split(text):
     """
     Generator that splits a string by spaces, leaving quoted phrases together.
@@ -205 +206 @@
     quote marks.
 
     >>> list(smart_split('This is "a person\'s" test.'))
-    ['This', 'is', '"a person\'s"', 'test.']
+    [u'This', u'is', u'"a person\'s"', u'test.']
+
+    Even if quoted content is found in the middle of a phrase, it is considered
+    part of the same phrase:
+
+    >>> text = '''with thelist|filter:'A B'|another:"Y Z" as var'''
+    >>> list(smart_split(text))
+    [u'with', u'thelist|filter:\'A B\'|another:"Y Z"', u'as', u'var']
     """
     text = force_unicode(text)
+    contents = []
     for bit in smart_split_re.finditer(text):
-        bit = bit.group(0)
-        if bit[0] == '"' and bit[-1] == '"':
-            yield '"' + bit[1:-1].replace('\\"', '"').replace('\\\\', '\\') + '"'
-        elif bit[0] == "'" and bit[-1] == "'":
-            yield "'" + bit[1:-1].replace("\\'", "'").replace("\\\\", "\\") + "'"
-        else:
-            yield bit
+        content = bit.group(1)
+        if content:
+            contents.append(content)
+        elif contents:
+            yield ''.join(contents)
+            contents = []
+    if contents:
+        yield ''.join(contents)
 smart_split = allow_lazy(smart_split, unicode)
 

tests/regressiontests/utils/tests.py

@@ -4 +4 @@
 
 from unittest import TestCase
 
-from django.utils import html, checksums
+from django.utils import html, checksums, text
 
 import timesince
 import datastructures
@@ -15 +15 @@
     'datastructures': datastructures,
 }
 
-class TestUtilsHtml(TestCase):
 
+class UtilsTestCase(TestCase):
+    
     def check_output(self, function, value, output=None):
         """
         Check that function(value) equals output.  If output is None,
@@ -26 +27 @@
             output = value
         self.assertEqual(function(value), output)
 
+    def check_generator_output(self, function, value, output=None):
+        """
+        Check that list(function(value)) equals output.  If output is None,
+        check that list(function(value)) equals value.
+        """
+        if output is None:
+            output = value
+        self.assertEqual(list(function(value)), output)
+
+class TestUtilsText(UtilsTestCase):
+
+    def test_smart_split(self):
+        f = text.smart_split
+        items = (
+            # Double quotes.
+            ('arg1|filter:"1 2"', [u'arg1|filter:"1 2"']),
+            ('arg1 arg2|filter:"1" arg3', [u'arg1', u'arg2|filter:"1"', u'arg3']),
+            ('arg1 arg2|filter:"1 2" arg3', [u'arg1', u'arg2|filter:"1 2"', u'arg3']),
+            ('arg1 arg2|filter:"1  2" arg3', [u'arg1', u'arg2|filter:"1  2"', u'arg3']),
+            ('arg1 arg2|filter:"1\t2" arg3', [u'arg1', u'arg2|filter:"1\t2"', u'arg3']),
+            # With escapes.
+            ('arg1 arg2|filter:"1\\"1" arg3', [u'arg1', u'arg2|filter:"1\\"1"', u'arg3']),
+            ('arg1 arg2|filter:"1\\"\'1" arg3', [u'arg1', u'arg2|filter:"1\\"\'1"', u'arg3']),
+            # Single quotes.
+            ("arg1|filter:'1 2'", [u"arg1|filter:'1 2'"]),
+            ("arg1 arg2|filter:'1' arg3", [u'arg1', u"arg2|filter:'1'", u'arg3']),
+            ("arg1 arg2|filter:'1 2' arg3", [u'arg1', u"arg2|filter:'1 2'", u'arg3']),
+            ("arg1 arg2|filter:'1  2' arg3", [u'arg1', u"arg2|filter:'1  2'", u'arg3']),
+            ("arg1 arg2|filter:'1\t2' arg3", [u'arg1', u"arg2|filter:'1\t2'", u'arg3']),
+            # With escapes.
+            ("arg1 arg2|filter:'1\\'1' arg3", [u'arg1', u"arg2|filter:'1\\'1'", u'arg3']),
+            ("arg1 arg2|filter:'1\\'\"1' arg3", [u'arg1', u"arg2|filter:'1\\'\"1'", u'arg3']),
+            # Mixed quotes and multiple filters.
+            ("""with thelist|filter:'A B'|another:"Y Z" as var""", [u'with', u'thelist|filter:\'A B\'|another:"Y Z"', u'as', u'var']),
+            # With escapes.
+            ("""with thelist|filter:'A\\' B'|another:"Y\\" Z" as var""", [u'with', u'thelist|filter:\'A\\\' B\'|another:"Y\\" Z"', u'as', u'var']),
+        )
+        for value, output in items:
+            self.check_generator_output(f, value, output)
+
+
+class TestUtilsHtml(UtilsTestCase):
+
     def test_escape(self):
         f = html.escape
         items = (
@@ -123 +167 @@
         for value, output in items:
             self.check_output(f, value, output)
 
-class TestUtilsChecksums(TestCase):
-
-    def check_output(self, function, value, output=None):
-        """
-        Check that function(value) equals output.  If output is None,
-        check that function(value) equals value.
-        """
-        if output is None:
-            output = value
-        self.assertEqual(function(value), output)
+class TestUtilsChecksums(UtilsTestCase):
 
     def test_luhn(self):
         f = checksums.luhn