Ticket #6271: smart_split.diff

django/utils/text.py

@@ -178 +178 @@
 ustring_re = re.compile(u"([\u0080-\uffff])")
 
 def javascript_quote(s, quote_double_quotes=False):
-
     def fix(match):
         return r"\u%04x" % ord(match.group(1))
 
@@ -196 +195 @@
     return str(ustring_re.sub(fix, s))
 javascript_quote = allow_lazy(javascript_quote, unicode)
 
-smart_split_re = re.compile('("(?:[^"\\\\]*(?:\\\\.[^"\\\\]*)*)"|\'(?:[^\'\\\\]*(?:\\\\.[^\'\\\\]*)*)\'|[^\\s]+)')
+smart_split_re = re.compile(r'("(?:[^"\\]*(?:\\.[^"\\]*)*)"'
+                            r"|'(?:[^'\\]*(?:\\.[^'\\]*)*)'"
+                            r'|[^\s"\']+|["\'])|\s+')
 def smart_split(text):
     """
     Generator that splits a string by spaces, leaving quoted phrases together.
@@ -205 +206 @@
     quote marks.
 
     >>> list(smart_split('This is "a person\'s" test.'))
-    ['This', 'is', '"a person\'s"', 'test.']
+    [u'This', u'is', u'"a person\'s"', u'test.']
+
+    Even if quoted content is found in the middle of a phrase, it is considered
+    part of the same phrase:
+
+    >>> text = '''with thelist|filter:'A B'|another:"Y Z" as var'''
+    >>> list(smart_split(text))
+    [u'with', u'thelist|filter:\'A B\'|another:"Y Z"', u'as', u'var']
     """
     text = force_unicode(text)
+    contents = []
     for bit in smart_split_re.finditer(text):
-        bit = bit.group(0)
-        if bit[0] == '"' and bit[-1] == '"':
-            yield '"' + bit[1:-1].replace('\\"', '"').replace('\\\\', '\\') + '"'
-        elif bit[0] == "'" and bit[-1] == "'":
-            yield "'" + bit[1:-1].replace("\\'", "'").replace("\\\\", "\\") + "'"
-        else:
-            yield bit
+        content = bit.group(1)
+        if content:
+            if content.startswith('"') and content.endswith('"'):
+                content = u'"%s"' % content[1:-1].replace('\\"', '"')\
+                                                 .replace('\\\\', '\\')
+            elif content.startswith("'") and content.endswith("'"):
+                content = u"'%s'" % content[1:-1].replace("\\'", "'")\
+                                                 .replace("\\\\", "\\")
+            contents.append(content)
+        elif contents:
+            yield ''.join(contents)
+            contents = []
+    if contents:
+        yield ''.join(contents)
 smart_split = allow_lazy(smart_split, unicode)