Code

Ticket #6271: smart_split.2.diff

File smart_split.2.diff, 5.6 KB (added by gwilson, 7 years ago)
Line 
1=== modified file 'django/utils/text.py'
2--- django/utils/text.py        2007-08-15 12:09:32 +0000
3+++ django/utils/text.py        2008-01-07 01:23:17 +0000
4@@ -178,7 +178,6 @@
5 ustring_re = re.compile(u"([\u0080-\uffff])")
6 
7 def javascript_quote(s, quote_double_quotes=False):
8-
9     def fix(match):
10         return r"\u%04x" % ord(match.group(1))
11 
12@@ -196,7 +195,9 @@
13     return str(ustring_re.sub(fix, s))
14 javascript_quote = allow_lazy(javascript_quote, unicode)
15 
16-smart_split_re = re.compile('("(?:[^"\\\\]*(?:\\\\.[^"\\\\]*)*)"|\'(?:[^\'\\\\]*(?:\\\\.[^\'\\\\]*)*)\'|[^\\s]+)')
17+smart_split_re = re.compile(r'("(?:[^"\\]*(?:\\.[^"\\]*)*)"'
18+                            r"|'(?:[^'\\]*(?:\\.[^'\\]*)*)'"
19+                            r'|[^\s"\']+|["\'])|\s+')
20 def smart_split(text):
21     """
22     Generator that splits a string by spaces, leaving quoted phrases together.
23@@ -205,16 +206,25 @@
24     quote marks.
25 
26     >>> list(smart_split('This is "a person\'s" test.'))
27-    ['This', 'is', '"a person\'s"', 'test.']
28+    [u'This', u'is', u'"a person\'s"', u'test.']
29+
30+    Even if quoted content is found in the middle of a phrase, it is considered
31+    part of the same phrase:
32+
33+    >>> text = '''with thelist|filter:'A B'|another:"Y Z" as var'''
34+    >>> list(smart_split(text))
35+    [u'with', u'thelist|filter:\'A B\'|another:"Y Z"', u'as', u'var']
36     """
37     text = force_unicode(text)
38+    contents = []
39     for bit in smart_split_re.finditer(text):
40-        bit = bit.group(0)
41-        if bit[0] == '"' and bit[-1] == '"':
42-            yield '"' + bit[1:-1].replace('\\"', '"').replace('\\\\', '\\') + '"'
43-        elif bit[0] == "'" and bit[-1] == "'":
44-            yield "'" + bit[1:-1].replace("\\'", "'").replace("\\\\", "\\") + "'"
45-        else:
46-            yield bit
47+        content = bit.group(1)
48+        if content:
49+            contents.append(content)
50+        elif contents:
51+            yield ''.join(contents)
52+            contents = []
53+    if contents:
54+        yield ''.join(contents)
55 smart_split = allow_lazy(smart_split, unicode)
56 
57
58=== modified file 'tests/regressiontests/utils/tests.py'
59--- tests/regressiontests/utils/tests.py        2007-11-29 20:09:54 +0000
60+++ tests/regressiontests/utils/tests.py        2008-01-07 01:19:34 +0000
61@@ -4,7 +4,7 @@
62 
63 from unittest import TestCase
64 
65-from django.utils import html, checksums
66+from django.utils import html, checksums, text
67 
68 import timesince
69 import datastructures
70@@ -15,8 +15,9 @@
71     'datastructures': datastructures,
72 }
73 
74-class TestUtilsHtml(TestCase):
75 
76+class UtilsTestCase(TestCase):
77+   
78     def check_output(self, function, value, output=None):
79         """
80         Check that function(value) equals output.  If output is None,
81@@ -26,6 +27,49 @@
82             output = value
83         self.assertEqual(function(value), output)
84 
85+    def check_generator_output(self, function, value, output=None):
86+        """
87+        Check that list(function(value)) equals output.  If output is None,
88+        check that list(function(value)) equals value.
89+        """
90+        if output is None:
91+            output = value
92+        self.assertEqual(list(function(value)), output)
93+
94+class TestUtilsText(UtilsTestCase):
95+
96+    def test_smart_split(self):
97+        f = text.smart_split
98+        items = (
99+            # Double quotes.
100+            ('arg1|filter:"1 2"', [u'arg1|filter:"1 2"']),
101+            ('arg1 arg2|filter:"1" arg3', [u'arg1', u'arg2|filter:"1"', u'arg3']),
102+            ('arg1 arg2|filter:"1 2" arg3', [u'arg1', u'arg2|filter:"1 2"', u'arg3']),
103+            ('arg1 arg2|filter:"1  2" arg3', [u'arg1', u'arg2|filter:"1  2"', u'arg3']),
104+            ('arg1 arg2|filter:"1\t2" arg3', [u'arg1', u'arg2|filter:"1\t2"', u'arg3']),
105+            # With escapes.
106+            ('arg1 arg2|filter:"1\\"1" arg3', [u'arg1', u'arg2|filter:"1\\"1"', u'arg3']),
107+            ('arg1 arg2|filter:"1\\"\'1" arg3', [u'arg1', u'arg2|filter:"1\\"\'1"', u'arg3']),
108+            # Single quotes.
109+            ("arg1|filter:'1 2'", [u"arg1|filter:'1 2'"]),
110+            ("arg1 arg2|filter:'1' arg3", [u'arg1', u"arg2|filter:'1'", u'arg3']),
111+            ("arg1 arg2|filter:'1 2' arg3", [u'arg1', u"arg2|filter:'1 2'", u'arg3']),
112+            ("arg1 arg2|filter:'1  2' arg3", [u'arg1', u"arg2|filter:'1  2'", u'arg3']),
113+            ("arg1 arg2|filter:'1\t2' arg3", [u'arg1', u"arg2|filter:'1\t2'", u'arg3']),
114+            # With escapes.
115+            ("arg1 arg2|filter:'1\\'1' arg3", [u'arg1', u"arg2|filter:'1\\'1'", u'arg3']),
116+            ("arg1 arg2|filter:'1\\'\"1' arg3", [u'arg1', u"arg2|filter:'1\\'\"1'", u'arg3']),
117+            # Mixed quotes and multiple filters.
118+            ("""with thelist|filter:'A B'|another:"Y Z" as var""", [u'with', u'thelist|filter:\'A B\'|another:"Y Z"', u'as', u'var']),
119+            # With escapes.
120+            ("""with thelist|filter:'A\\' B'|another:"Y\\" Z" as var""", [u'with', u'thelist|filter:\'A\\\' B\'|another:"Y\\" Z"', u'as', u'var']),
121+        )
122+        for value, output in items:
123+            self.check_generator_output(f, value, output)
124+
125+
126+class TestUtilsHtml(UtilsTestCase):
127+
128     def test_escape(self):
129         f = html.escape
130         items = (
131@@ -123,16 +167,7 @@
132         for value, output in items:
133             self.check_output(f, value, output)
134 
135-class TestUtilsChecksums(TestCase):
136-
137-    def check_output(self, function, value, output=None):
138-        """
139-        Check that function(value) equals output.  If output is None,
140-        check that function(value) equals value.
141-        """
142-        if output is None:
143-            output = value
144-        self.assertEqual(function(value), output)
145+class TestUtilsChecksums(UtilsTestCase):
146 
147     def test_luhn(self):
148         f = checksums.luhn
149