diff --git a/AUTHORS b/AUTHORS index ea0b019..969ba49 100644 --- a/AUTHORS +++ b/AUTHORS @@ -292,6 +292,7 @@ answer newbie questions, and generally made Django that much better: Ian G. Kelly Niall Kelly Ryan Kelly + Stephen Kelly Thomas Kerpe Wiley Kestner Ossama M. Khayat diff --git a/django/conf/global_settings.py b/django/conf/global_settings.py index 13f7991..05bb413 100644 --- a/django/conf/global_settings.py +++ b/django/conf/global_settings.py @@ -209,6 +209,9 @@ TEMPLATE_CONTEXT_PROCESSORS = ( # Output to use in template system for invalid (e.g. misspelled) variables. TEMPLATE_STRING_IF_INVALID = '' +# Remove the empty lines that had template tags on them. +TEMPLATE_STRIP_LEADING_WHITESPACE = False + # Default email address to use for various automated correspondence from # the site managers. DEFAULT_FROM_EMAIL = 'webmaster@localhost' diff --git a/django/template/base.py b/django/template/base.py index 89bc909..0964f5d 100644 --- a/django/template/base.py +++ b/django/template/base.py @@ -55,10 +55,58 @@ UNKNOWN_SOURCE = '' # match a variable or block tag and capture the entire tag, including start/end # delimiters -tag_re = (re.compile('(%s.*?%s|%s.*?%s|%s.*?%s)' % - (re.escape(BLOCK_TAG_START), re.escape(BLOCK_TAG_END), - re.escape(VARIABLE_TAG_START), re.escape(VARIABLE_TAG_END), - re.escape(COMMENT_TAG_START), re.escape(COMMENT_TAG_END)))) +tag_re = re.compile(''' + ( + %(BLOCK_TAG_START)s.*?%(BLOCK_TAG_END)s # block start + inner + end + | %(VARIABLE_TAG_START)s.*?%(VARIABLE_TAG_END)s # variable start + inner + end + | %(COMMENT_TAG_START)s.*?%(COMMENT_TAG_END)s # comment start + inner + end + ) + ''' % { + 'BLOCK_TAG_START': re.escape(BLOCK_TAG_START), + 'BLOCK_TAG_END': re.escape(BLOCK_TAG_END), + 'VARIABLE_TAG_START': re.escape(VARIABLE_TAG_START), + 'VARIABLE_TAG_END': re.escape(VARIABLE_TAG_END), + 'COMMENT_TAG_START': re.escape(COMMENT_TAG_START), + 'COMMENT_TAG_END': re.escape(COMMENT_TAG_END), + }, re.VERBOSE) + +strip_leading_whitespace_tag_re = re.compile(''' + ( + [\n\r]{1} # 1 vertical white space (the one starting this line) + [ \t]*? # any number of tabs or spaces + (?: # group but don't match + %(BLOCK_TAG_START)s + [^\n\r%(BLOCK_TAG_START)s%(BLOCK_TAG_END)s]*? # anything *not* vertical whitespace or + %(BLOCK_TAG_END)s # opening/closing block tag + | + %(COMMENT_TAG_START)s + [^\n\r%(COMMENT_TAG_START)s%(COMMENT_TAG_END)s]*? # anything *not* vertical whitespace or + %(COMMENT_TAG_END)s # opening/closing comment tag + | + %(VARIABLE_TAG_START)s + [^\n\r%(VARIABLE_TAG_START)s%(VARIABLE_TAG_END)s]*? # anything *not* vertical whitespace or + %(VARIABLE_TAG_END)s # opening/closing variable tag + ) + (?= # Match this only if the previous matched + [ \t]*? + [\n\r]+ + )| + %(BLOCK_TAG_START)s.*?%(BLOCK_TAG_END)s # block start + inner + end + | %(VARIABLE_TAG_START)s.*?%(VARIABLE_TAG_END)s # variable start + inner + end + | %(COMMENT_TAG_START)s.*?%(COMMENT_TAG_END)s # comment start + inner + end + )''' % { + 'BLOCK_TAG_START': re.escape(BLOCK_TAG_START), + 'BLOCK_TAG_END': re.escape(BLOCK_TAG_END), + 'VARIABLE_TAG_START': re.escape(VARIABLE_TAG_START), + 'VARIABLE_TAG_END': re.escape(VARIABLE_TAG_END), + 'COMMENT_TAG_START': re.escape(COMMENT_TAG_START), + 'COMMENT_TAG_END': re.escape(COMMENT_TAG_END), + }, re.VERBOSE) + +# find the command within a block tag +block_command_re = re.compile('%s(?P.*?)%s' % (re.escape(BLOCK_TAG_START), re.escape(BLOCK_TAG_END))) +comment_command_re = re.compile('%s(?P.*?)%s' % (re.escape(COMMENT_TAG_START), re.escape(COMMENT_TAG_END))) +variable_command_re = re.compile('%s(?P.*?)%s' % (re.escape(VARIABLE_TAG_START), re.escape(VARIABLE_TAG_END))) # global dictionary of libraries that have been loaded using get_library libraries = {} @@ -179,12 +227,13 @@ class Token(object): split.append(bit) return split + class Lexer(object): def __init__(self, template_string, origin): self.template_string = template_string self.origin = origin self.lineno = 1 - self.verbatim = False + self.verbatim = False # or 'endverbatim' or 'endverbatim %s' % block def tokenize(self): """ @@ -192,7 +241,8 @@ class Lexer(object): """ in_tag = False result = [] - for bit in tag_re.split(self.template_string): + tag_splitter = strip_leading_whitespace_tag_re if settings.TEMPLATE_STRIP_LEADING_WHITESPACE else tag_re + for bit in tag_splitter.split(self.template_string): if bit: result.append(self.create_token(bit, in_tag)) in_tag = not in_tag @@ -204,7 +254,8 @@ class Lexer(object): If in_tag is True, we are processing something that matched a tag, otherwise it should be treated as a literal string. """ - if in_tag and token_string.startswith(BLOCK_TAG_START): + # Extract ``block_content``, check for verbatim tag + if in_tag and token_string.lstrip().startswith(BLOCK_TAG_START): # The [2:-2] ranges below strip off *_TAG_START and *_TAG_END. # We could do len(BLOCK_TAG_START) to be more "correct", but we've # hard-coded the 2s here for performance. And it's not like @@ -212,24 +263,39 @@ class Lexer(object): block_content = token_string[2:-2].strip() if self.verbatim and block_content == self.verbatim: self.verbatim = False + + # Create ``token`` object... if in_tag and not self.verbatim: - if token_string.startswith(VARIABLE_TAG_START): - token = Token(TOKEN_VAR, token_string[2:-2].strip()) - elif token_string.startswith(BLOCK_TAG_START): + # ...by processing a... + if token_string.lstrip().startswith(VARIABLE_TAG_START): + # ...variable tag. + token_struct = variable_command_re.search(token_string) + token_string = token_struct.group('token_string') + token = Token(TOKEN_VAR, token_string.strip()) + elif token_string.lstrip().startswith(BLOCK_TAG_START): + # ...block tag. if block_content[:9] in ('verbatim', 'verbatim '): self.verbatim = 'end%s' % block_content - token = Token(TOKEN_BLOCK, block_content) - elif token_string.startswith(COMMENT_TAG_START): + token_struct = block_command_re.search(token_string) + token_string = token_struct.group('token_string') + token = Token(TOKEN_BLOCK, token_string.strip()) + elif token_string.lstrip().startswith(COMMENT_TAG_START): + # ...comment tag. content = '' + token_struct = comment_command_re.search(token_string) + token_string = token_struct.group('token_string') if token_string.find(TRANSLATOR_COMMENT_MARK): - content = token_string[2:-2].strip() + content = token_string.strip() token = Token(TOKEN_COMMENT, content) else: + # ...from a string literal. token = Token(TOKEN_TEXT, token_string) + token.lineno = self.lineno self.lineno += token_string.count('\n') return token + class Parser(object): def __init__(self, tokens): self.tokens = tokens diff --git a/django/template/debug.py b/django/template/debug.py index 6167403..3df6681 100644 --- a/django/template/debug.py +++ b/django/template/debug.py @@ -1,4 +1,5 @@ -from django.template.base import Lexer, Parser, tag_re, NodeList, VariableNode, TemplateSyntaxError +from django.conf import settings +from django.template.base import Lexer, Parser, tag_re, strip_leading_whitespace_tag_re, NodeList, VariableNode, TemplateSyntaxError from django.utils.encoding import force_unicode from django.utils.html import escape from django.utils.safestring import SafeData, EscapeData @@ -13,7 +14,8 @@ class DebugLexer(Lexer): def tokenize(self): "Return a list of tokens from a given template_string" result, upto = [], 0 - for match in tag_re.finditer(self.template_string): + tag_splitter = strip_leading_whitespace_tag_re if settings.TEMPLATE_STRIP_LEADING_WHITESPACE else tag_re + for match in tag_splitter.finditer(self.template_string): start, end = match.span() if start > upto: result.append(self.create_token(self.template_string[upto:start], (upto, start), False)) diff --git a/django/template/defaulttags.py b/django/template/defaulttags.py index 83b72e1..2ca094d 100644 --- a/django/template/defaulttags.py +++ b/django/template/defaulttags.py @@ -14,9 +14,11 @@ from django.template.base import (Node, NodeList, Template, Context, Library, VARIABLE_ATTRIBUTE_SEPARATOR, get_library, token_kwargs, kwarg_re) from django.template.smartif import IfParser, Literal from django.template.defaultfilters import date +from django.utils import timezone from django.utils.encoding import smart_unicode from django.utils.safestring import mark_safe -from django.utils import timezone +from django.utils.text import smart_split + register = Library() diff --git a/docs/ref/settings.txt b/docs/ref/settings.txt index 627aa50..ce9779e 100644 --- a/docs/ref/settings.txt +++ b/docs/ref/settings.txt @@ -1956,6 +1956,63 @@ Default: ``''`` (Empty string) Output, as a string, that the template system should use for invalid (e.g. misspelled) variables. See :ref:`invalid-template-variables`.. +.. setting:: TEMPLATE_STRIP_LEADING_WHITESPACE + +TEMPLATE_STRIP_LEADING_WHITESPACE +--------------------------------- + +.. versionadded:: 1.5 + +Default: ``False`` + +Whether to strip leading whitespace on template lines containing only template +syntax. + +This template code:: +

My list

+ + {# This is a comment #} + {# It describes the loop below #} + {# Each line appears in the output as an empty line #} +
    + {% for item in items %} + {# Description of a list item #} +
  • {{ item }}
    • + {% endfor %} +
+ +Normally evaluates to:: +

My list

+ + + + +
    + + +
  • item 1
    • + +
  • item 2
    • + +
  • item 3
    • + +
+ +However, if TEMPLATE_STRIP_LEADING_WHITESPACE is set to True, lines which +contain only one template tag, variable or comment no not cause a newline +to be inserted in the output:: + +

My list

+ +
    +
  • item 1
    • +
  • item 2
    • +
  • item 3
    • +
+ +Trailing whitespace does not get stripped, and lines which contain more +than one template tag will not have their leading whitespace trimmed. + .. setting:: TEST_RUNNER TEST_RUNNER diff --git a/tests/regressiontests/templates/tests.py b/tests/regressiontests/templates/tests.py index 35d0122..a69fa23 100644 --- a/tests/regressiontests/templates/tests.py +++ b/tests/regressiontests/templates/tests.py @@ -387,6 +387,211 @@ class Templates(unittest.TestCase): except TemplateSyntaxError as e: self.assertEqual(e.args[0], "Invalid block tag: 'endblock', expected 'elif', 'else' or 'endif'") + def test_insignificant_whitespace(self): + whitespace_tests = { + # The test tuple contents is (template_content, context_args, stripped_output, unstripped_output) + # Tags on their own line should collapse the newline before them + # Trailing newline is not removed + # Leading whitespace before single template tag + 'insignificant-whitespace01': ('\n {% templatetag openblock %}\n', {}, '{%\n', + '\n {%\n'), + 'insignificant-whitespace02': ('\n{% templatetag openblock %}\n', {}, '{%\n', + '\n{%\n'), + 'insignificant-whitespace03': ('{% templatetag openblock %}\n', {}, '{%\n', + '{%\n'), + 'insignificant-whitespace04': ('\n\t \t {% templatetag openblock %}\n', {}, '{%\n', + '\n\t \t {%\n'), + # Leading whitespace with text before single template tag + 'insignificant-whitespace05': ('\n some\ttext {% templatetag openblock %}\n', {}, '\n some\ttext {%\n', + '\n some\ttext {%\n'), + # Leading line with text before single template tag + 'insignificant-whitespace06': ('\n some\ttext\n {% templatetag openblock %}\n', {}, '\n some\ttext{%\n', + '\n some\ttext\n {%\n'), + 'insignificant-whitespace07': ('\n some\ttext \n \t {% templatetag openblock %}\n', {}, '\n some\ttext {%\n', + '\n some\ttext \n \t {%\n'), + # whitespace leading /before/ the newline is not stripped. + 'insignificant-whitespace08': ('\n some\ttext \t \n {% templatetag openblock %}\n', {}, '\n some\ttext \t {%\n', + '\n some\ttext \t \n {%\n'), + # Multiple text lines before tag + 'insignificant-whitespace09': ('\n some\ntext \t \n {% templatetag openblock %}\n', {}, '\n some\ntext \t {%\n', + '\n some\ntext \t \n {%\n'), + 'insignificant-whitespace10': ('\n some \t \n \t text \t \n {% templatetag openblock %}\n', {}, '\n some \t \n \t text \t {%\n', + '\n some \t \n \t text \t \n {%\n'), + # Leading whitespace before tag, some text after + 'insignificant-whitespace11': ('\n \t {% templatetag openblock %} some text\n', {}, '\n \t {% some text\n', + '\n \t {% some text\n'), + # Leading whitespace before tag, some text with trailing whitespace after + 'insignificant-whitespace12': ('\n \t {% templatetag openblock %} some text \t \n', {}, '\n \t {% some text \t \n', + '\n \t {% some text \t \n'), + # Whitespace after tag is not removed + 'insignificant-whitespace13': ('\n \t {% templatetag openblock %} \t \n \t some text \t \n', {}, '{% \t \n \t some text \t \n', + '\n \t {% \t \n \t some text \t \n'), + # Multiple lines of leading whitespace. Only one leading newline is removed + 'insignificant-whitespace14': ('\n\n\n{% templatetag openblock %}\n some text\n', {}, '\n\n{%\n some text\n', + '\n\n\n{%\n some text\n'), + # Trailing whitespace after tag + 'insignificant-whitespace15': ('\n\n\n{% templatetag openblock %}\t \t \t\n some text\n', {}, '\n\n{%\t \t \t\n some text\n', + '\n\n\n{%\t \t \t\n some text\n'), + # Removable newline followed by leading whitespace + 'insignificant-whitespace16': ('\n\n\n\t \t \t{% templatetag openblock %}\n some text\n', {}, '\n\n{%\n some text\n', + '\n\n\n\t \t \t{%\n some text\n'), + # Removable leading whitespace and trailing whitespace + 'insignificant-whitespace17': ('\n\n\n\t \t \t{% templatetag openblock %}\t \t \t\n some text\n', {}, '\n\n{%\t \t \t\n some text\n', + '\n\n\n\t \t \t{%\t \t \t\n some text\n'), + # Multiple lines of trailing whitespace. No trailing newline is removed. + 'insignificant-whitespace18': ('\n{% templatetag openblock %}\n\n\n some text\n', {}, '{%\n\n\n some text\n', + '\n{%\n\n\n some text\n'), + 'insignificant-whitespace19': ('\n{% templatetag openblock %}\t \n\n\n some text\n', {}, '{%\t \n\n\n some text\n', + '\n{%\t \n\n\n some text\n'), + # Consecutive trimmed lines with tags strips one newline each + 'insignificant-whitespace20': ( + '\n{% templatetag openblock %}\n{% templatetag openblock %}\n{% templatetag openblock %}\n some text\n' + , {}, '{%{%{%\n some text\n', + '\n{%\n{%\n{%\n some text\n'), + # Consecutive trimmed lines with tags strips one newline each. Intermediate newlines are preserved + 'insignificant-whitespace21': ( + '\n\n{% templatetag openblock %}\n\n{% templatetag openblock %}\n\n{% templatetag openblock %}\n\n some text\n' + , {}, '\n{%\n{%\n{%\n\n some text\n', + '\n\n{%\n\n{%\n\n{%\n\n some text\n'), + # Consecutive trimmed lines with tags strips one newline each. Leading whitespace is stripped but trailing is not + 'insignificant-whitespace22': ( + '\n\n\t {% templatetag openblock %}\t \n\n\t {% templatetag openblock %}\t \n\n\t {% templatetag openblock %}\t \n some text\n' + , {}, '\n{%\t \n{%\t \n{%\t \n some text\n', + '\n\n\t {%\t \n\n\t {%\t \n\n\t {%\t \n some text\n'), + # Consecutive trimmed lines with tags strips one newline each. Intermediate whitespace is stripped + 'insignificant-whitespace23': ( + '\n\t {% templatetag openblock %}\t \n\t {% templatetag openblock %}\t \n\t {% templatetag openblock %}\t \n some text\n' + , {}, '{%\t {%\t {%\t \n some text\n', + '\n\t {%\t \n\t {%\t \n\t {%\t \n some text\n'), + # Intermediate whitespace on one line is preserved + # Consecutive tags on one line do not have intermediate whitespace or leading whitespace stripped + 'insignificant-whitespace24': ( + '\n\t {% templatetag openblock %}\t \t {% templatetag openblock %}\t \t {% templatetag openblock %}\t \n some text\n' + , {}, '\n\t {%\t \t {%\t \t {%\t \n some text\n', + '\n\t {%\t \t {%\t \t {%\t \n some text\n'), + # Still, only one leading newline is removed. + 'insignificant-whitespace25': ( + '\n\n {% templatetag openblock %}\n \t {% templatetag openblock %}\n \t {% templatetag openblock %}\n some text\n' + , {}, '\n{%{%{%\n some text\n', + '\n\n {%\n \t {%\n \t {%\n some text\n'), + # Lines with {# comments #} have the same stripping behavior + 'insignificant-whitespace26': ( + '\n\n {% templatetag openblock %}\n \t {# some comment #}\n some text\n' + , {}, '\n{%\n some text\n', + '\n\n {%\n \t \n some text\n'), + # Only {# comments #} + 'insignificant-whitespace27': ( + '\n\n {# a comment #}\n \t {# some comment #}\n some text\n' + , {}, '\n\n some text\n', + '\n\n \n \t \n some text\n'), + # Consecutive newlines with tags and comments + 'insignificant-whitespace28': ( + '\n\t {% templatetag openblock %}\t \n\t {# some comment #}\t \n\t {% templatetag openblock %}\t \n some text\n' + , {}, '{%\t \t {%\t \n some text\n', + '\n\t {%\t \n\t \t \n\t {%\t \n some text\n'), + + # Lines with only {{ values }} have the same stripping behavior + 'insignificant-whitespace29': ( + '\n {% templatetag openblock %}\t\n \t {{ spam }}\t \n \t {% templatetag openblock %}\t \n some text\n' + , {"spam" : "ham"}, '{%\tham\t {%\t \n some text\n', + '\n {%\t\n \t ham\t \n \t {%\t \n some text\n'), + 'insignificant-whitespace30': ( + '\n\n {% templatetag openblock %}\t\n\n \t {{ spam }}\t \n\n \t {% templatetag openblock %}\t \n some text\n' + , {"spam" : "ham"}, '\n{%\t\nham\t \n{%\t \n some text\n', + '\n\n {%\t\n\n \t ham\t \n\n \t {%\t \n some text\n'), + ## Leading whitespace not stripped when followed by anything. See insignificant-whitespace24 + 'insignificant-whitespace31': ( + '\n {% templatetag openblock %}\t \t {{ spam }}\t \t {% templatetag openblock %}\t \n some text\n' + , {"spam" : "ham"}, '\n {%\t \t ham\t \t {%\t \n some text\n', + '\n {%\t \t ham\t \t {%\t \n some text\n'), + # {{ value }} {% tag %} {{ value }} this time + 'insignificant-whitespace32': ( + '\n {{ spam }}\t\n \t {% templatetag openblock %}\t \n \t {{ spam }}\t \n some text\n' + , {"spam" : "ham"}, 'ham\t{%\t ham\t \n some text\n', + '\n ham\t\n \t {%\t \n \t ham\t \n some text\n'), + + # Invalid stuff is still invalid + # Newlines inside begin-end tokens, even in {# comments #}, make it not a tag. + 'insignificant-whitespace33': ( + '\n\n {# \n{% templatetag openblock #}\t \n some text\n' + , {}, '\n\n {# \n{% templatetag openblock #}\t \n some text\n', + '\n\n {# \n{% templatetag openblock #}\t \n some text\n'), + # Complete comment matching tags on one line are processed + 'insignificant-whitespace34': ( + '\n\n {# \n{# templatetag openblock #}\t \n some text\n' + , {}, '\n\n {# \t \n some text\n', + '\n\n {# \n\t \n some text\n'), + 'insignificant-whitespace35': ( + '\n\n {# \n{# templatetag openblock\n #}\t \n some text\n' + , {}, '\n\n {# \n{# templatetag openblock\n #}\t \n some text\n', + '\n\n {# \n{# templatetag openblock\n #}\t \n some text\n'), + 'insignificant-whitespace36': ( + '\n\n {# \n{{ some comment #}\t \n some text\n' + , {}, '\n\n {# \n{{ some comment #}\t \n some text\n', + '\n\n {# \n{{ some comment #}\t \n some text\n'), + 'insignificant-whitespace37': ( + '\n\n {# \n \t {% templatetag openblock #}\t \n some text\n' + , {}, '\n\n {# \n \t {% templatetag openblock #}\t \n some text\n', + '\n\n {# \n \t {% templatetag openblock #}\t \n some text\n'), + 'insignificant-whitespace38': ( + "\n\n {# templatetag openblock #\n}\t \n some text\n" + , {}, "\n\n {# templatetag openblock #\n}\t \n some text\n", + "\n\n {# templatetag openblock #\n}\t \n some text\n" ), + 'insignificant-whitespace39': ( + "\n\n {% templatetag openblock %\n}\t \n some text\n" + , {}, "\n\n {% templatetag openblock %\n}\t \n some text\n", + "\n\n {% templatetag openblock %\n}\t \n some text\n" ), + 'insignificant-whitespace40': ( + "\n\n {{ templatetag openblock }\n}\t \n some text\n" + , {}, "\n\n {{ templatetag openblock }\n}\t \n some text\n", + "\n\n {{ templatetag openblock }\n}\t \n some text\n" ), + 'insignificant-whitespace41': ( + "\n\n {\n# {# templatetag openblock #}\t \n some text\n" + , {}, "\n\n {\n# \t \n some text\n", + "\n\n {\n# \t \n some text\n"), + 'insignificant-whitespace42': ( + "\n\n {\n {# templatetag openblock #}\t \n some text\n" + , {}, "\n\n {\t \n some text\n", + "\n\n {\n \t \n some text\n"), + 'insignificant-whitespace43': ( + "\n{{# foo #};{# bar #}\n" + , {}, "\n{;\n", + "\n{;\n"), + } + tests = whitespace_tests.items() + tests.sort() + + # Register our custom template loader. + def test_whitespace_loader(template_name, template_dirs=None): + "A custom template loader that loads the unit-test templates." + try: + return (whitespace_tests[template_name][0] , "test:%s" % template_name) + except KeyError: + raise template.TemplateDoesNotExist, template_name + + old_template_loaders = loader.template_source_loaders + loader.template_source_loaders = [test_whitespace_loader] + + failures = [] + + old_strip_leading_whitespace = settings.TEMPLATE_STRIP_LEADING_WHITESPACE + + for name, vals in tests: + for strip_leading_whitespace in (True, False): + settings.TEMPLATE_STRIP_LEADING_WHITESPACE = strip_leading_whitespace + test_template = loader.get_template(name) + result = vals[2] if strip_leading_whitespace else vals[3] + output = self.render(test_template, vals) + + if output != result: + failures.append("Whitespace test: %s -- FAILED. Expected %r, got %r" % (name, result, output)) + + loader.template_source_loaders = old_template_loaders + settings.TEMPLATE_STRIP_LEADING_WHITESPACE = old_strip_leading_whitespace + + self.assertEqual(failures, [], "Tests failed:\n%s\n%s" % + ('-'*70, ("\n%s\n" % ('-'*70)).join(failures))) + def test_templates(self): template_tests = self.get_template_tests() filter_tests = filters.get_filter_tests()