diff --git a/django/template/base.py b/django/template/base.py index d934e05..1d41d2a 100644 --- a/django/template/base.py +++ b/django/template/base.py @@ -1,4 +1,3 @@ -import imp import re from inspect import getargspec @@ -6,7 +5,7 @@ from django.conf import settings from django.template.context import Context, RequestContext, ContextPopException from django.utils.importlib import import_module from django.utils.itercompat import is_iterable -from django.utils.functional import curry, Promise +from django.utils.functional import curry from django.utils.text import smart_split, unescape_string_literal, get_text_list from django.utils.encoding import smart_unicode, force_unicode, smart_str from django.utils.translation import ugettext as _ @@ -42,9 +41,43 @@ ALLOWED_VARIABLE_CHARS = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ01 UNKNOWN_SOURCE = '' # match a variable or block tag and capture the entire tag, including start/end delimiters -tag_re = re.compile('(%s.*?%s|%s.*?%s|%s.*?%s)' % (re.escape(BLOCK_TAG_START), re.escape(BLOCK_TAG_END), - re.escape(VARIABLE_TAG_START), re.escape(VARIABLE_TAG_END), - re.escape(COMMENT_TAG_START), re.escape(COMMENT_TAG_END))) +tag_re = re.compile(''' + ( + [\n\r\v\f]{1} # 1 vertical white space (the one starting this line) + [ \t]*? # any number of tabs or spaces + (?: # group but don't match + %(BLOCK_TAG_START)s + [^\n\r\v\f%(BLOCK_TAG_START)s%(BLOCK_TAG_END)s]*? # anything *not* vertical whitespace or + %(BLOCK_TAG_END)s # opening/closing block tag + | + %(COMMENT_TAG_START)s + [^\n\r\v\f%(COMMENT_TAG_START)s%(COMMENT_TAG_END)s]*? # anything *not* vertical whitespace or + %(COMMENT_TAG_END)s # opening/closing comment tag + | + %(VARIABLE_TAG_START)s + [^\n\r\v\f%(VARIABLE_TAG_START)s%(VARIABLE_TAG_END)s]*? # anything *not* vertical whitespace or + %(VARIABLE_TAG_END)s # opening/closing variable tag + ) + (?= # Match this only if the previous matched + [ \t]*? + [\n\r\v\f]+ + ) + | %(BLOCK_TAG_START)s.*?%(BLOCK_TAG_END)s # block start + inner + end + | %(VARIABLE_TAG_START)s.*?%(VARIABLE_TAG_END)s # variable start + inner + end + | %(COMMENT_TAG_START)s.*?%(COMMENT_TAG_END)s # comment start + inner + end + ) + ''' % { + 'BLOCK_TAG_START': re.escape(BLOCK_TAG_START), + 'BLOCK_TAG_END': re.escape(BLOCK_TAG_END), + 'VARIABLE_TAG_START': re.escape(VARIABLE_TAG_START), + 'VARIABLE_TAG_END': re.escape(VARIABLE_TAG_END), + 'COMMENT_TAG_START': re.escape(COMMENT_TAG_START), + 'COMMENT_TAG_END': re.escape(COMMENT_TAG_END), + }, re.VERBOSE) +# find the command within a block tag +block_command_re = re.compile('%s(?P.*?)%s' % (re.escape(BLOCK_TAG_START), re.escape(BLOCK_TAG_END))) +comment_command_re = re.compile('%s(?P.*?)%s' % (re.escape(COMMENT_TAG_START), re.escape(COMMENT_TAG_END))) +variable_command_re = re.compile('%s(?P.*?)%s' % (re.escape(VARIABLE_TAG_START), re.escape(VARIABLE_TAG_END))) # global dictionary of libraries that have been loaded using get_library libraries = {} @@ -184,12 +217,18 @@ class Lexer(object): otherwise it should be treated as a literal string. """ if in_tag: - if token_string.startswith(VARIABLE_TAG_START): - token = Token(TOKEN_VAR, token_string[len(VARIABLE_TAG_START):-len(VARIABLE_TAG_END)].strip()) - elif token_string.startswith(BLOCK_TAG_START): - token = Token(TOKEN_BLOCK, token_string[len(BLOCK_TAG_START):-len(BLOCK_TAG_END)].strip()) - elif token_string.startswith(COMMENT_TAG_START): + if token_string.lstrip().startswith(VARIABLE_TAG_START): + token_struct = variable_command_re.search(token_string) + token_string = token_struct.group('token_string') + token = Token(TOKEN_VAR, token_string.strip()) + elif token_string.lstrip().startswith(BLOCK_TAG_START): + token_struct = block_command_re.search(token_string) + token_string = token_struct.group('token_string') + token = Token(TOKEN_BLOCK, token_string.strip()) + elif token_string.lstrip().startswith(COMMENT_TAG_START): content = '' + token_struct = comment_command_re.search(token_string) + token_string = token_struct.group('token_string') if token_string.find(TRANSLATOR_COMMENT_MARK): content = token_string[len(COMMENT_TAG_START):-len(COMMENT_TAG_END)].strip() token = Token(TOKEN_COMMENT, content) diff --git a/tests/regressiontests/templates/tests.py b/tests/regressiontests/templates/tests.py index 475f923..fce1a93 100644 --- a/tests/regressiontests/templates/tests.py +++ b/tests/regressiontests/templates/tests.py @@ -1251,6 +1251,117 @@ class Templates(unittest.TestCase): 'templatetag11': ('{% templatetag opencomment %}', {}, '{#'), 'templatetag12': ('{% templatetag closecomment %}', {}, '#}'), + ### Remove insignificant whitespace ####################################### + # Tags on their own line should collapse the newline before them + # Trailing newline is not removed + # Leading whitespace before single template tag + 'templatetag-whitespace01': ('\n {% templatetag openblock %}\n', {}, '{%\n'), + 'templatetag-whitespace02': ('\n{% templatetag openblock %}\n', {}, '{%\n'), + 'templatetag-whitespace03': ('{% templatetag openblock %}\n', {}, '{%\n'), + 'templatetag-whitespace04': ('\n\t \t {% templatetag openblock %}\n', {}, '{%\n'), + # Leading whitespace with text before single template tag + 'templatetag-whitespace05': ('\n some\ttext {% templatetag openblock %}\n', {}, '\n some\ttext {%\n'), + # Leading line with text before single template tag + 'templatetag-whitespace06': ('\n some\ttext\n {% templatetag openblock %}\n', {}, '\n some\ttext{%\n'), + 'templatetag-whitespace07': ('\n some\ttext \n \t {% templatetag openblock %}\n', {}, '\n some\ttext {%\n'), + # whitespace leading /before/ the newline is not stripped. + 'templatetag-whitespace08': ('\n some\ttext \t \n {% templatetag openblock %}\n', {}, '\n some\ttext \t {%\n'), + # Multiple text lines before tag + 'templatetag-whitespace09': ('\n some\ntext \t \n {% templatetag openblock %}\n', {}, '\n some\ntext \t {%\n'), + 'templatetag-whitespace10': ('\n some \t \n \t text \t \n {% templatetag openblock %}\n', {}, '\n some \t \n \t text \t {%\n'), + # Leading whitespace before tag, some text after + 'templatetag-whitespace11': ('\n \t {% templatetag openblock %} some text\n', {}, '\n \t {% some text\n'), + # Leading whitespace before tag, some text with trailing whitespace after + 'templatetag-whitespace12': ('\n \t {% templatetag openblock %} some text \t \n', {}, '\n \t {% some text \t \n'), + # Whitespace after tag is not removed + 'templatetag-whitespace13': ('\n \t {% templatetag openblock %} \t \n \t some text \t \n', {}, '{% \t \n \t some text \t \n'), + # Multiple lines of leading whitespace. Only one leading newline is removed + 'templatetag-whitespace14': ('\n\n\n{% templatetag openblock %}\n some text\n', {}, '\n\n{%\n some text\n'), + # Trailing whitespace after tag + 'templatetag-whitespace15': ('\n\n\n{% templatetag openblock %}\t \t \t\n some text\n', {}, '\n\n{%\t \t \t\n some text\n'), + # Removable newline followed by leading whitespace + 'templatetag-whitespace16': ('\n\n\n\t \t \t{% templatetag openblock %}\n some text\n', {}, '\n\n{%\n some text\n'), + # Removable leading whitespace and trailing whitespace + 'templatetag-whitespace17': ('\n\n\n\t \t \t{% templatetag openblock %}\t \t \t\n some text\n', {}, '\n\n{%\t \t \t\n some text\n'), + # Multiple lines of trailing whitespace. No trailing newline is removed. + 'templatetag-whitespace18': ('\n{% templatetag openblock %}\n\n\n some text\n', {}, '{%\n\n\n some text\n'), + 'templatetag-whitespace19': ('\n{% templatetag openblock %}\t \n\n\n some text\n', {}, '{%\t \n\n\n some text\n'), + # Consecutive trimmed lines with tags strips one newline each + 'templatetag-whitespace20': ( + '\n{% templatetag openblock %}\n{% templatetag openblock %}\n{% templatetag openblock %}\n some text\n' + , {}, '{%{%{%\n some text\n'), + # Consecutive trimmed lines with tags strips one newline each. Intermediate newlines are preserved + 'templatetag-whitespace21': ( + '\n\n{% templatetag openblock %}\n\n{% templatetag openblock %}\n\n{% templatetag openblock %}\n\n some text\n' + , {}, '\n{%\n{%\n{%\n\n some text\n'), + # Consecutive trimmed lines with tags strips one newline each. Leading whitespace is stripped but trailing is not + 'templatetag-whitespace22': ( + '\n\n\t {% templatetag openblock %}\t \n\n\t {% templatetag openblock %}\t \n\n\t {% templatetag openblock %}\t \n some text\n' + , {}, '\n{%\t \n{%\t \n{%\t \n some text\n'), + # Consecutive trimmed lines with tags strips one newline each. Intermediate whitespace is stripped + 'templatetag-whitespace23': ( + '\n\t {% templatetag openblock %}\t \n\t {% templatetag openblock %}\t \n\t {% templatetag openblock %}\t \n some text\n' + , {}, '{%\t {%\t {%\t \n some text\n'), + # Intermediate whitespace on one line is preserved + # Consecutive tags on one line do not have intermediate whitespace or leading whitespace stripped + 'templatetag-whitespace24': ( + '\n\t {% templatetag openblock %}\t \t {% templatetag openblock %}\t \t {% templatetag openblock %}\t \n some text\n' + , {}, '\n\t {%\t \t {%\t \t {%\t \n some text\n'), + # Still, only one leading newline is removed. + 'templatetag-whitespace25': ( + '\n\n {% templatetag openblock %}\n \t {% templatetag openblock %}\n \t {% templatetag openblock %}\n some text\n' + , {}, '\n{%{%{%\n some text\n'), + # Lines with {# comments #} have the same stripping behavior + 'templatetag-whitespace26': ( + '\n\n {% templatetag openblock %}\n \t {# some comment #}\n some text\n' + , {}, '\n{%\n some text\n'), + # Only {# comments #} + 'templatetag-whitespace27': ( + '\n\n {# a comment #}\n \t {# some comment #}\n some text\n' + , {}, '\n\n some text\n'), + # Consecutive newlines with tags and comments + 'templatetag-whitespace28': ( + '\n\t {% templatetag openblock %}\t \n\t {# some comment #}\t \n\t {% templatetag openblock %}\t \n some text\n' + , {}, '{%\t \t {%\t \n some text\n'), + + # Lines with only {{ values }} have the same stripping behavior + 'templatetag-whitespace29': ( + '\n {% templatetag openblock %}\t\n \t {{ spam }}\t \n \t {% templatetag openblock %}\t \n some text\n' + , {"spam" : "ham"}, '{%\tham\t {%\t \n some text\n'), + 'templatetag-whitespace30': ( + '\n\n {% templatetag openblock %}\t\n\n \t {{ spam }}\t \n\n \t {% templatetag openblock %}\t \n some text\n' + , {"spam" : "ham"}, '\n{%\t\nham\t \n{%\t \n some text\n'), + # Leading whitespace not stripped when followed by anything. See templatetag-whitespace24 + 'templatetag-whitespace31': ( + '\n {% templatetag openblock %}\t \t {{ spam }}\t \t {% templatetag openblock %}\t \n some text\n' + , {"spam" : "ham"}, '\n {%\t \t ham\t \t {%\t \n some text\n'), + # {{ value }} {% tag %} {{ value }} this time + 'templatetag-whitespace32': ( + '\n {{ spam }}\t\n \t {% templatetag openblock %}\t \n \t {{ spam }}\t \n some text\n' + , {"spam" : "ham"}, 'ham\t{%\t ham\t \n some text\n'), + + # Invalid stuff is still invalid + # Newlines inside begin-end tokens, even in {# comments #}, make it not a tag. + 'templatetag-whitespace33': ( + '\n\n {# \n{% templatetag openblock #}\t \n some text\n' + , {}, '\n\n {# \n{% templatetag openblock #}\t \n some text\n'), + # Complete comment matching tags on one line are processed + 'templatetag-whitespace34': ( + '\n\n {# \n{# templatetag openblock #}\t \n some text\n' + , {}, '\n\n {# \t \n some text\n'), + 'templatetag-whitespace35': ( + '\n\n {# \n{# templatetag openblock\n #}\t \n some text\n' + , {}, '\n\n {# \n{# templatetag openblock\n #}\t \n some text\n'), + 'templatetag-whitespace36': ( + '\n\n {# \n{{ some comment #}\t \n some text\n' + , {}, '\n\n {# \n{{ some comment #}\t \n some text\n'), + 'templatetag-whitespace37': ( + '\n\n {# \n \t {% templatetag openblock #}\t \n some text\n' + , {}, '\n\n {# \n \t {% templatetag openblock #}\t \n some text\n'), + + ### WIDTHRATIO TAG ######################################################## + 'widthratio01': ('{% widthratio a b 0 %}', {'a':50,'b':100}, '0'), + 'widthratio02': ('{% widthratio a b 100 %}', {'a':0,'b':0}, ''), ### WIDTHRATIO TAG ######################################################## 'widthratio01': ('{% widthratio a b 0 %}', {'a':50,'b':100}, '0'), 'widthratio02': ('{% widthratio a b 100 %}', {'a':0,'b':0}, ''),