Ticket #6271: smart_split.2.diff

File smart_split.2.diff, 5.6 KB (added by gwilson, 8 years ago)
  • django/utils/text.py

    === modified file 'django/utils/text.py'
     
    178178ustring_re = re.compile(u"([\u0080-\uffff])")
    179179
    180180def javascript_quote(s, quote_double_quotes=False):
    181 
    182181    def fix(match):
    183182        return r"\u%04x" % ord(match.group(1))
    184183
     
    196195    return str(ustring_re.sub(fix, s))
    197196javascript_quote = allow_lazy(javascript_quote, unicode)
    198197
    199 smart_split_re = re.compile('("(?:[^"\\\\]*(?:\\\\.[^"\\\\]*)*)"|\'(?:[^\'\\\\]*(?:\\\\.[^\'\\\\]*)*)\'|[^\\s]+)')
     198smart_split_re = re.compile(r'("(?:[^"\\]*(?:\\.[^"\\]*)*)"'
     199                            r"|'(?:[^'\\]*(?:\\.[^'\\]*)*)'"
     200                            r'|[^\s"\']+|["\'])|\s+')
    200201def smart_split(text):
    201202    """
    202203    Generator that splits a string by spaces, leaving quoted phrases together.
     
    205206    quote marks.
    206207
    207208    >>> list(smart_split('This is "a person\'s" test.'))
    208     ['This', 'is', '"a person\'s"', 'test.']
     209    [u'This', u'is', u'"a person\'s"', u'test.']
     210
     211    Even if quoted content is found in the middle of a phrase, it is considered
     212    part of the same phrase:
     213
     214    >>> text = '''with thelist|filter:'A B'|another:"Y Z" as var'''
     215    >>> list(smart_split(text))
     216    [u'with', u'thelist|filter:\'A B\'|another:"Y Z"', u'as', u'var']
    209217    """
    210218    text = force_unicode(text)
     219    contents = []
    211220    for bit in smart_split_re.finditer(text):
    212         bit = bit.group(0)
    213         if bit[0] == '"' and bit[-1] == '"':
    214             yield '"' + bit[1:-1].replace('\\"', '"').replace('\\\\', '\\') + '"'
    215         elif bit[0] == "'" and bit[-1] == "'":
    216             yield "'" + bit[1:-1].replace("\\'", "'").replace("\\\\", "\\") + "'"
    217         else:
    218             yield bit
     221        content = bit.group(1)
     222        if content:
     223            contents.append(content)
     224        elif contents:
     225            yield ''.join(contents)
     226            contents = []
     227    if contents:
     228        yield ''.join(contents)
    219229smart_split = allow_lazy(smart_split, unicode)
    220230
  • tests/regressiontests/utils/tests.py

    === modified file 'tests/regressiontests/utils/tests.py'
     
    44
    55from unittest import TestCase
    66
    7 from django.utils import html, checksums
     7from django.utils import html, checksums, text
    88
    99import timesince
    1010import datastructures
     
    1515    'datastructures': datastructures,
    1616}
    1717
    18 class TestUtilsHtml(TestCase):
    1918
     19class UtilsTestCase(TestCase):
     20   
    2021    def check_output(self, function, value, output=None):
    2122        """
    2223        Check that function(value) equals output.  If output is None,
     
    2627            output = value
    2728        self.assertEqual(function(value), output)
    2829
     30    def check_generator_output(self, function, value, output=None):
     31        """
     32        Check that list(function(value)) equals output.  If output is None,
     33        check that list(function(value)) equals value.
     34        """
     35        if output is None:
     36            output = value
     37        self.assertEqual(list(function(value)), output)
     38
     39class TestUtilsText(UtilsTestCase):
     40
     41    def test_smart_split(self):
     42        f = text.smart_split
     43        items = (
     44            # Double quotes.
     45            ('arg1|filter:"1 2"', [u'arg1|filter:"1 2"']),
     46            ('arg1 arg2|filter:"1" arg3', [u'arg1', u'arg2|filter:"1"', u'arg3']),
     47            ('arg1 arg2|filter:"1 2" arg3', [u'arg1', u'arg2|filter:"1 2"', u'arg3']),
     48            ('arg1 arg2|filter:"1  2" arg3', [u'arg1', u'arg2|filter:"1  2"', u'arg3']),
     49            ('arg1 arg2|filter:"1\t2" arg3', [u'arg1', u'arg2|filter:"1\t2"', u'arg3']),
     50            # With escapes.
     51            ('arg1 arg2|filter:"1\\"1" arg3', [u'arg1', u'arg2|filter:"1\\"1"', u'arg3']),
     52            ('arg1 arg2|filter:"1\\"\'1" arg3', [u'arg1', u'arg2|filter:"1\\"\'1"', u'arg3']),
     53            # Single quotes.
     54            ("arg1|filter:'1 2'", [u"arg1|filter:'1 2'"]),
     55            ("arg1 arg2|filter:'1' arg3", [u'arg1', u"arg2|filter:'1'", u'arg3']),
     56            ("arg1 arg2|filter:'1 2' arg3", [u'arg1', u"arg2|filter:'1 2'", u'arg3']),
     57            ("arg1 arg2|filter:'1  2' arg3", [u'arg1', u"arg2|filter:'1  2'", u'arg3']),
     58            ("arg1 arg2|filter:'1\t2' arg3", [u'arg1', u"arg2|filter:'1\t2'", u'arg3']),
     59            # With escapes.
     60            ("arg1 arg2|filter:'1\\'1' arg3", [u'arg1', u"arg2|filter:'1\\'1'", u'arg3']),
     61            ("arg1 arg2|filter:'1\\'\"1' arg3", [u'arg1', u"arg2|filter:'1\\'\"1'", u'arg3']),
     62            # Mixed quotes and multiple filters.
     63            ("""with thelist|filter:'A B'|another:"Y Z" as var""", [u'with', u'thelist|filter:\'A B\'|another:"Y Z"', u'as', u'var']),
     64            # With escapes.
     65            ("""with thelist|filter:'A\\' B'|another:"Y\\" Z" as var""", [u'with', u'thelist|filter:\'A\\\' B\'|another:"Y\\" Z"', u'as', u'var']),
     66        )
     67        for value, output in items:
     68            self.check_generator_output(f, value, output)
     69
     70
     71class TestUtilsHtml(UtilsTestCase):
     72
    2973    def test_escape(self):
    3074        f = html.escape
    3175        items = (
     
    123167        for value, output in items:
    124168            self.check_output(f, value, output)
    125169
    126 class TestUtilsChecksums(TestCase):
    127 
    128     def check_output(self, function, value, output=None):
    129         """
    130         Check that function(value) equals output.  If output is None,
    131         check that function(value) equals value.
    132         """
    133         if output is None:
    134             output = value
    135         self.assertEqual(function(value), output)
     170class TestUtilsChecksums(UtilsTestCase):
    136171
    137172    def test_luhn(self):
    138173        f = checksums.luhn
Back to Top