Ticket #2594: 2594.diff

File 2594.diff, 14.2 KB (added by robhudson, 4 years ago)

Patched against trunk (and template shuffling), removed trailing white space, and adjusted to add comments and make pass with steveire's tests.

  • django/template/base.py

    diff --git a/django/template/base.py b/django/template/base.py
    index d934e05..1d41d2a 100644
    a b  
    1 import imp
    21import re
    32from inspect import getargspec
    43
    from django.conf import settings 
    65from django.template.context import Context, RequestContext, ContextPopException
    76from django.utils.importlib import import_module
    87from django.utils.itercompat import is_iterable
    9 from django.utils.functional import curry, Promise
     8from django.utils.functional import curry
    109from django.utils.text import smart_split, unescape_string_literal, get_text_list
    1110from django.utils.encoding import smart_unicode, force_unicode, smart_str
    1211from django.utils.translation import ugettext as _
    ALLOWED_VARIABLE_CHARS = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ01 
    4241UNKNOWN_SOURCE = '<unknown source>'
    4342
    4443# match a variable or block tag and capture the entire tag, including start/end delimiters
    45 tag_re = re.compile('(%s.*?%s|%s.*?%s|%s.*?%s)' % (re.escape(BLOCK_TAG_START), re.escape(BLOCK_TAG_END),
    46                                           re.escape(VARIABLE_TAG_START), re.escape(VARIABLE_TAG_END),
    47                                           re.escape(COMMENT_TAG_START), re.escape(COMMENT_TAG_END)))
     44tag_re = re.compile('''
     45    (
     46        [\n\r\v\f]{1} # 1 vertical white space (the one starting this line)
     47        [ \t]*?       # any number of tabs or spaces
     48        (?:           # group but don't match
     49            %(BLOCK_TAG_START)s
     50            [^\n\r\v\f%(BLOCK_TAG_START)s%(BLOCK_TAG_END)s]*? # anything *not* vertical whitespace or
     51            %(BLOCK_TAG_END)s                                 # opening/closing block tag
     52        |
     53            %(COMMENT_TAG_START)s
     54            [^\n\r\v\f%(COMMENT_TAG_START)s%(COMMENT_TAG_END)s]*? # anything *not* vertical whitespace or
     55            %(COMMENT_TAG_END)s                                   # opening/closing comment tag
     56        |
     57            %(VARIABLE_TAG_START)s
     58            [^\n\r\v\f%(VARIABLE_TAG_START)s%(VARIABLE_TAG_END)s]*? # anything *not* vertical whitespace or
     59            %(VARIABLE_TAG_END)s                                    # opening/closing variable tag
     60        )
     61        (?=                      # Match this only if the previous matched
     62            [ \t]*?
     63            [\n\r\v\f]+
     64        )
     65        | %(BLOCK_TAG_START)s.*?%(BLOCK_TAG_END)s        # block start + inner + end
     66        | %(VARIABLE_TAG_START)s.*?%(VARIABLE_TAG_END)s  # variable start + inner + end
     67        | %(COMMENT_TAG_START)s.*?%(COMMENT_TAG_END)s    # comment start + inner + end
     68    )
     69    ''' % {
     70        'BLOCK_TAG_START': re.escape(BLOCK_TAG_START),
     71        'BLOCK_TAG_END': re.escape(BLOCK_TAG_END),
     72        'VARIABLE_TAG_START': re.escape(VARIABLE_TAG_START),
     73        'VARIABLE_TAG_END': re.escape(VARIABLE_TAG_END),
     74        'COMMENT_TAG_START': re.escape(COMMENT_TAG_START),
     75        'COMMENT_TAG_END': re.escape(COMMENT_TAG_END),
     76    }, re.VERBOSE)
     77# find the command within a block tag
     78block_command_re = re.compile('%s(?P<token_string>.*?)%s' % (re.escape(BLOCK_TAG_START), re.escape(BLOCK_TAG_END)))
     79comment_command_re = re.compile('%s(?P<token_string>.*?)%s' % (re.escape(COMMENT_TAG_START), re.escape(COMMENT_TAG_END)))
     80variable_command_re = re.compile('%s(?P<token_string>.*?)%s' % (re.escape(VARIABLE_TAG_START), re.escape(VARIABLE_TAG_END)))
    4881
    4982# global dictionary of libraries that have been loaded using get_library
    5083libraries = {}
    class Lexer(object): 
    184217        otherwise it should be treated as a literal string.
    185218        """
    186219        if in_tag:
    187             if token_string.startswith(VARIABLE_TAG_START):
    188                 token = Token(TOKEN_VAR, token_string[len(VARIABLE_TAG_START):-len(VARIABLE_TAG_END)].strip())
    189             elif token_string.startswith(BLOCK_TAG_START):
    190                 token = Token(TOKEN_BLOCK, token_string[len(BLOCK_TAG_START):-len(BLOCK_TAG_END)].strip())
    191             elif token_string.startswith(COMMENT_TAG_START):
     220            if token_string.lstrip().startswith(VARIABLE_TAG_START):
     221                token_struct = variable_command_re.search(token_string)
     222                token_string = token_struct.group('token_string')
     223                token = Token(TOKEN_VAR, token_string.strip())
     224            elif token_string.lstrip().startswith(BLOCK_TAG_START):
     225                token_struct = block_command_re.search(token_string)
     226                token_string = token_struct.group('token_string')
     227                token = Token(TOKEN_BLOCK, token_string.strip())
     228            elif token_string.lstrip().startswith(COMMENT_TAG_START):
    192229                content = ''
     230                token_struct = comment_command_re.search(token_string)
     231                token_string = token_struct.group('token_string')
    193232                if token_string.find(TRANSLATOR_COMMENT_MARK):
    194233                    content = token_string[len(COMMENT_TAG_START):-len(COMMENT_TAG_END)].strip()
    195234                token = Token(TOKEN_COMMENT, content)
  • tests/regressiontests/templates/tests.py

    diff --git a/tests/regressiontests/templates/tests.py b/tests/regressiontests/templates/tests.py
    index 475f923..fce1a93 100644
    a b class Templates(unittest.TestCase): 
    12511251            'templatetag11': ('{% templatetag opencomment %}', {}, '{#'),
    12521252            'templatetag12': ('{% templatetag closecomment %}', {}, '#}'),
    12531253
     1254            ### Remove insignificant whitespace #######################################
     1255            # Tags on their own line should collapse the newline before them
     1256            # Trailing newline is not removed
     1257            # Leading whitespace before single template tag
     1258            'templatetag-whitespace01': ('\n {% templatetag openblock %}\n', {}, '{%\n'),
     1259            'templatetag-whitespace02': ('\n{% templatetag openblock %}\n', {}, '{%\n'),
     1260            'templatetag-whitespace03': ('{% templatetag openblock %}\n', {}, '{%\n'),
     1261            'templatetag-whitespace04': ('\n\t \t {% templatetag openblock %}\n', {}, '{%\n'),
     1262            # Leading whitespace with text before single template tag
     1263            'templatetag-whitespace05': ('\n some\ttext {% templatetag openblock %}\n', {}, '\n some\ttext {%\n'),
     1264            # Leading line with text before single template tag
     1265            'templatetag-whitespace06': ('\n some\ttext\n {% templatetag openblock %}\n', {}, '\n some\ttext{%\n'),
     1266            'templatetag-whitespace07': ('\n some\ttext \n \t {% templatetag openblock %}\n', {}, '\n some\ttext {%\n'),
     1267            # whitespace leading /before/ the newline is not stripped.
     1268            'templatetag-whitespace08': ('\n some\ttext \t \n {% templatetag openblock %}\n', {}, '\n some\ttext \t {%\n'),
     1269            # Multiple text lines before tag
     1270            'templatetag-whitespace09': ('\n some\ntext \t \n {% templatetag openblock %}\n', {}, '\n some\ntext \t {%\n'),
     1271            'templatetag-whitespace10': ('\n some \t \n \t text \t \n {% templatetag openblock %}\n', {}, '\n some \t \n \t text \t {%\n'),
     1272            # Leading whitespace before tag, some text after
     1273            'templatetag-whitespace11': ('\n   \t {% templatetag openblock %} some text\n', {}, '\n   \t {% some text\n'),
     1274            # Leading whitespace before tag, some text with trailing whitespace after
     1275            'templatetag-whitespace12': ('\n   \t {% templatetag openblock %} some text  \t \n', {}, '\n   \t {% some text  \t \n'),
     1276            # Whitespace after tag is not removed
     1277            'templatetag-whitespace13': ('\n \t {% templatetag openblock %} \t \n \t some text  \t \n', {}, '{% \t \n \t some text  \t \n'),
     1278            # Multiple lines of leading whitespace. Only one leading newline is removed
     1279            'templatetag-whitespace14': ('\n\n\n{% templatetag openblock %}\n some text\n', {}, '\n\n{%\n some text\n'),
     1280            # Trailing whitespace after tag
     1281            'templatetag-whitespace15': ('\n\n\n{% templatetag openblock %}\t \t \t\n some text\n', {}, '\n\n{%\t \t \t\n some text\n'),
     1282            # Removable newline followed by leading whitespace
     1283            'templatetag-whitespace16': ('\n\n\n\t \t \t{% templatetag openblock %}\n some text\n', {}, '\n\n{%\n some text\n'),
     1284            # Removable leading whitespace and trailing whitespace
     1285            'templatetag-whitespace17': ('\n\n\n\t \t \t{% templatetag openblock %}\t \t \t\n some text\n', {}, '\n\n{%\t \t \t\n some text\n'),
     1286            # Multiple lines of trailing whitespace. No trailing newline is removed.
     1287            'templatetag-whitespace18': ('\n{% templatetag openblock %}\n\n\n some text\n', {}, '{%\n\n\n some text\n'),
     1288            'templatetag-whitespace19': ('\n{% templatetag openblock %}\t \n\n\n some text\n', {}, '{%\t \n\n\n some text\n'),
     1289            # Consecutive trimmed lines with tags strips one newline each
     1290            'templatetag-whitespace20': (
     1291                '\n{% templatetag openblock %}\n{% templatetag openblock %}\n{% templatetag openblock %}\n some text\n'
     1292                , {}, '{%{%{%\n some text\n'),
     1293            # Consecutive trimmed lines with tags strips one newline each. Intermediate newlines are preserved
     1294            'templatetag-whitespace21': (
     1295                '\n\n{% templatetag openblock %}\n\n{% templatetag openblock %}\n\n{% templatetag openblock %}\n\n some text\n'
     1296                , {}, '\n{%\n{%\n{%\n\n some text\n'),
     1297            # Consecutive trimmed lines with tags strips one newline each. Leading whitespace is stripped but trailing is not
     1298            'templatetag-whitespace22': (
     1299                '\n\n\t {% templatetag openblock %}\t \n\n\t {% templatetag openblock %}\t \n\n\t {% templatetag openblock %}\t \n some text\n'
     1300                , {}, '\n{%\t \n{%\t \n{%\t \n some text\n'),
     1301            # Consecutive trimmed lines with tags strips one newline each. Intermediate whitespace is stripped
     1302            'templatetag-whitespace23': (
     1303                '\n\t {% templatetag openblock %}\t \n\t {% templatetag openblock %}\t \n\t {% templatetag openblock %}\t \n some text\n'
     1304                , {}, '{%\t {%\t {%\t \n some text\n'),
     1305            # Intermediate whitespace on one line is preserved
     1306            # Consecutive tags on one line do not have intermediate whitespace or leading whitespace stripped
     1307            'templatetag-whitespace24': (
     1308                '\n\t {% templatetag openblock %}\t \t {% templatetag openblock %}\t \t {% templatetag openblock %}\t \n some text\n'
     1309                , {}, '\n\t {%\t \t {%\t \t {%\t \n some text\n'),
     1310            # Still, only one leading newline is removed.
     1311            'templatetag-whitespace25': (
     1312                '\n\n {% templatetag openblock %}\n \t {% templatetag openblock %}\n \t {% templatetag openblock %}\n some text\n'
     1313                , {}, '\n{%{%{%\n some text\n'),
     1314            # Lines with {# comments #} have the same stripping behavior
     1315            'templatetag-whitespace26': (
     1316                '\n\n {% templatetag openblock %}\n \t {# some comment #}\n some text\n'
     1317                , {}, '\n{%\n some text\n'),
     1318            # Only {# comments #}
     1319            'templatetag-whitespace27': (
     1320                '\n\n {# a comment #}\n \t {# some comment #}\n some text\n'
     1321                , {}, '\n\n some text\n'),
     1322            # Consecutive newlines with tags and comments
     1323            'templatetag-whitespace28': (
     1324                '\n\t {% templatetag openblock %}\t \n\t {# some comment #}\t \n\t {% templatetag openblock %}\t \n some text\n'
     1325                , {}, '{%\t \t {%\t \n some text\n'),
     1326
     1327            # Lines with only {{ values }} have the same stripping behavior
     1328            'templatetag-whitespace29': (
     1329                '\n {% templatetag openblock %}\t\n \t {{ spam }}\t \n \t {% templatetag openblock %}\t \n some text\n'
     1330                , {"spam" : "ham"}, '{%\tham\t {%\t \n some text\n'),
     1331            'templatetag-whitespace30': (
     1332                '\n\n {% templatetag openblock %}\t\n\n \t {{ spam }}\t \n\n \t {% templatetag openblock %}\t \n some text\n'
     1333                , {"spam" : "ham"}, '\n{%\t\nham\t \n{%\t \n some text\n'),
     1334            # Leading whitespace not stripped when followed by anything. See templatetag-whitespace24
     1335            'templatetag-whitespace31': (
     1336                '\n {% templatetag openblock %}\t \t {{ spam }}\t \t {% templatetag openblock %}\t \n some text\n'
     1337                , {"spam" : "ham"}, '\n {%\t \t ham\t \t {%\t \n some text\n'),
     1338            #  {{ value }} {% tag %} {{ value }} this time
     1339            'templatetag-whitespace32': (
     1340                '\n {{ spam }}\t\n \t {% templatetag openblock %}\t \n \t {{ spam }}\t \n some text\n'
     1341                , {"spam" : "ham"}, 'ham\t{%\t ham\t \n some text\n'),
     1342
     1343            # Invalid stuff is still invalid
     1344            # Newlines inside begin-end tokens, even in {# comments #}, make it not a tag.
     1345            'templatetag-whitespace33': (
     1346                '\n\n {# \n{% templatetag openblock #}\t \n some text\n'
     1347                , {}, '\n\n {# \n{% templatetag openblock #}\t \n some text\n'),
     1348            # Complete comment matching tags on one line are processed
     1349            'templatetag-whitespace34': (
     1350                '\n\n {# \n{# templatetag openblock #}\t \n some text\n'
     1351                , {}, '\n\n {# \t \n some text\n'),
     1352            'templatetag-whitespace35': (
     1353                '\n\n {# \n{# templatetag openblock\n #}\t \n some text\n'
     1354                , {}, '\n\n {# \n{# templatetag openblock\n #}\t \n some text\n'),
     1355            'templatetag-whitespace36': (
     1356                '\n\n {# \n{{ some comment #}\t \n some text\n'
     1357                , {}, '\n\n {# \n{{ some comment #}\t \n some text\n'),
     1358            'templatetag-whitespace37': (
     1359                '\n\n {# \n \t {% templatetag openblock #}\t \n some text\n'
     1360                , {}, '\n\n {# \n \t {% templatetag openblock #}\t \n some text\n'),
     1361
     1362             ### WIDTHRATIO TAG ########################################################
     1363             'widthratio01': ('{% widthratio a b 0 %}', {'a':50,'b':100}, '0'),
     1364             'widthratio02': ('{% widthratio a b 100 %}', {'a':0,'b':0}, ''),
    12541365            ### WIDTHRATIO TAG ########################################################
    12551366            'widthratio01': ('{% widthratio a b 0 %}', {'a':50,'b':100}, '0'),
    12561367            'widthratio02': ('{% widthratio a b 100 %}', {'a':0,'b':0}, ''),
Back to Top