commit ffaecbd5cf0387868af908f89e0b6695de755174 Author: Stephen Kelly Date: Thu Jan 20 18:52:36 2011 +0100 Handle whitespace better in the template system. diff --git a/AUTHORS b/AUTHORS index ffcc34d..32dba07 100644 --- a/AUTHORS +++ b/AUTHORS @@ -262,6 +262,7 @@ answer newbie questions, and generally made Django that much better: Ian G. Kelly Niall Kelly Ryan Kelly + Stephen Kelly Thomas Kerpe Wiley Kestner Ossama M. Khayat diff --git a/django/conf/global_settings.py b/django/conf/global_settings.py index 0017f46..e5c0dbf 100644 --- a/django/conf/global_settings.py +++ b/django/conf/global_settings.py @@ -11,6 +11,7 @@ gettext_noop = lambda s: s DEBUG = False TEMPLATE_DEBUG = False +TEMPLATE_STRIP_LEADING_WHITESPACE = False # Whether the framework should propagate raw exceptions rather than catching # them. This is useful under some testing siutations and should never be used diff --git a/django/template/base.py b/django/template/base.py index 1440869..9211ebf 100644 --- a/django/template/base.py +++ b/django/template/base.py @@ -1,4 +1,3 @@ -import imp import re from inspect import getargspec @@ -6,7 +5,7 @@ from django.conf import settings from django.template.context import Context, RequestContext, ContextPopException from django.utils.importlib import import_module from django.utils.itercompat import is_iterable -from django.utils.functional import curry, Promise +from django.utils.functional import curry from django.utils.text import smart_split, unescape_string_literal, get_text_list from django.utils.encoding import smart_unicode, force_unicode, smart_str from django.utils.translation import ugettext as _ @@ -41,10 +40,60 @@ ALLOWED_VARIABLE_CHARS = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ01 # (e.g. strings) UNKNOWN_SOURCE = '' + # match a variable or block tag and capture the entire tag, including start/end delimiters -tag_re = re.compile('(%s.*?%s|%s.*?%s|%s.*?%s)' % (re.escape(BLOCK_TAG_START), re.escape(BLOCK_TAG_END), - re.escape(VARIABLE_TAG_START), re.escape(VARIABLE_TAG_END), - re.escape(COMMENT_TAG_START), re.escape(COMMENT_TAG_END))) +tag_re = re.compile(''' + ( + %(BLOCK_TAG_START)s.*?%(BLOCK_TAG_END)s # block start + inner + end + | %(VARIABLE_TAG_START)s.*?%(VARIABLE_TAG_END)s # variable start + inner + end + | %(COMMENT_TAG_START)s.*?%(COMMENT_TAG_END)s # comment start + inner + end + ) + ''' % { + 'BLOCK_TAG_START': re.escape(BLOCK_TAG_START), + 'BLOCK_TAG_END': re.escape(BLOCK_TAG_END), + 'VARIABLE_TAG_START': re.escape(VARIABLE_TAG_START), + 'VARIABLE_TAG_END': re.escape(VARIABLE_TAG_END), + 'COMMENT_TAG_START': re.escape(COMMENT_TAG_START), + 'COMMENT_TAG_END': re.escape(COMMENT_TAG_END), + }, re.VERBOSE) + +strip_leading_whitespace_tag_re = re.compile(''' + ( + [\n\r]{1} # 1 vertical white space (the one starting this line) + [ \t]*? # any number of tabs or spaces + (?: # group but don't match + %(BLOCK_TAG_START)s + [^\n\r%(BLOCK_TAG_START)s%(BLOCK_TAG_END)s]*? # anything *not* vertical whitespace or + %(BLOCK_TAG_END)s # opening/closing block tag + | + %(COMMENT_TAG_START)s + [^\n\r%(COMMENT_TAG_START)s%(COMMENT_TAG_END)s]*? # anything *not* vertical whitespace or + %(COMMENT_TAG_END)s # opening/closing comment tag + | + %(VARIABLE_TAG_START)s + [^\n\r%(VARIABLE_TAG_START)s%(VARIABLE_TAG_END)s]*? # anything *not* vertical whitespace or + %(VARIABLE_TAG_END)s # opening/closing variable tag + ) + (?= # Match this only if the previous matched + [ \t]*? + [\n\r]+ + )| + %(BLOCK_TAG_START)s.*?%(BLOCK_TAG_END)s # block start + inner + end + | %(VARIABLE_TAG_START)s.*?%(VARIABLE_TAG_END)s # variable start + inner + end + | %(COMMENT_TAG_START)s.*?%(COMMENT_TAG_END)s # comment start + inner + end + )''' % { + 'BLOCK_TAG_START': re.escape(BLOCK_TAG_START), + 'BLOCK_TAG_END': re.escape(BLOCK_TAG_END), + 'VARIABLE_TAG_START': re.escape(VARIABLE_TAG_START), + 'VARIABLE_TAG_END': re.escape(VARIABLE_TAG_END), + 'COMMENT_TAG_START': re.escape(COMMENT_TAG_START), + 'COMMENT_TAG_END': re.escape(COMMENT_TAG_END), + }, re.VERBOSE) + +# find the command within a block tag +block_command_re = re.compile('%s(?P.*?)%s' % (re.escape(BLOCK_TAG_START), re.escape(BLOCK_TAG_END))) +comment_command_re = re.compile('%s(?P.*?)%s' % (re.escape(COMMENT_TAG_START), re.escape(COMMENT_TAG_END))) +variable_command_re = re.compile('%s(?P.*?)%s' % (re.escape(VARIABLE_TAG_START), re.escape(VARIABLE_TAG_END))) # global dictionary of libraries that have been loaded using get_library libraries = {} @@ -171,7 +220,8 @@ class Lexer(object): "Return a list of tokens from a given template_string." in_tag = False result = [] - for bit in tag_re.split(self.template_string): + tag_splitter = strip_leading_whitespace_tag_re if settings.TEMPLATE_STRIP_LEADING_WHITESPACE else tag_re + for bit in tag_splitter.split(self.template_string): if bit: result.append(self.create_token(bit, in_tag)) in_tag = not in_tag @@ -184,12 +234,18 @@ class Lexer(object): otherwise it should be treated as a literal string. """ if in_tag: - if token_string.startswith(VARIABLE_TAG_START): - token = Token(TOKEN_VAR, token_string[len(VARIABLE_TAG_START):-len(VARIABLE_TAG_END)].strip()) - elif token_string.startswith(BLOCK_TAG_START): - token = Token(TOKEN_BLOCK, token_string[len(BLOCK_TAG_START):-len(BLOCK_TAG_END)].strip()) - elif token_string.startswith(COMMENT_TAG_START): + if token_string.lstrip().startswith(VARIABLE_TAG_START): + token_struct = variable_command_re.search(token_string) + token_string = token_struct.group('token_string') + token = Token(TOKEN_VAR, token_string.strip()) + elif token_string.lstrip().startswith(BLOCK_TAG_START): + token_struct = block_command_re.search(token_string) + token_string = token_struct.group('token_string') + token = Token(TOKEN_BLOCK, token_string.strip()) + elif token_string.lstrip().startswith(COMMENT_TAG_START): content = '' + token_struct = comment_command_re.search(token_string) + token_string = token_struct.group('token_string') if token_string.find(TRANSLATOR_COMMENT_MARK): content = token_string[len(COMMENT_TAG_START):-len(COMMENT_TAG_END)].strip() token = Token(TOKEN_COMMENT, content) diff --git a/django/template/defaulttags.py b/django/template/defaulttags.py index 0662b5f..13d4759 100644 --- a/django/template/defaulttags.py +++ b/django/template/defaulttags.py @@ -11,6 +11,7 @@ from django.template.smartif import IfParser, Literal from django.conf import settings from django.utils.encoding import smart_str, smart_unicode from django.utils.safestring import mark_safe +from django.utils.text import smart_split register = Library() # Regex for token keyword arguments @@ -47,7 +48,7 @@ def token_kwargs(bits, parser, support_legacy=False): kwargs = {} while bits: - if kwarg_format: + if kwarg_format: match = kwarg_re.match(bits[0]) if not match or not match.group(1): return kwargs diff --git a/docs/ref/settings.txt b/docs/ref/settings.txt index 87a71fe..4addda6 100644 --- a/docs/ref/settings.txt +++ b/docs/ref/settings.txt @@ -1755,6 +1755,61 @@ misspelled) variables. See :ref:`invalid-template-variables`.. .. setting:: TEST_RUNNER +TEMPLATE_STRIP_LEADING_WHITESPACE +--------------------------------- + +.. versionadded:: 1.3 + +Default: ``False`` + +Whether to strip leading whitespace on template lines containing only template +syntax. + +This template code:: +

My list

+ + {# This is a comment #} + {# It describes the loop below #} + {# Each line appears in the output as an empty line #} +
    + {% for item in items %} + {# Description of a list item #} +
  • {{ item }}
    • + {% endfor %} +
+ +Normally evaluates to:: +

My list

+ + + + +
    + + +
  • item 1
    • + +
  • item 2
    • + +
  • item 3
    • + +
+ +However, if TEMPLATE_STRIP_LEADING_WHITESPACE is set to True, lines which +contain only one template tag, variable or comment no not cause a newline +to be inserted in the output:: + +

My list

+ +
    +
  • item 1
    • +
  • item 2
    • +
  • item 3
    • +
+ +Trailing whitespace does not get stripped, and lines which contain more +than one template tag will not have their leading whitespace trimmed. + TEST_RUNNER ----------- diff --git a/tests/regressiontests/templates/tests.py b/tests/regressiontests/templates/tests.py index 99776e6..fb176fe 100644 --- a/tests/regressiontests/templates/tests.py +++ b/tests/regressiontests/templates/tests.py @@ -340,6 +340,211 @@ class Templates(unittest.TestCase): except TemplateSyntaxError, e: self.assertEquals(e.args[0], "Invalid block tag: 'endblock', expected 'else' or 'endif'") + def test_insignificant_whitespace(self): + whitespace_tests = { + # The test tuple contents is (template_content, context_args, stripped_output, unstripped_output) + # Tags on their own line should collapse the newline before them + # Trailing newline is not removed + # Leading whitespace before single template tag + 'insignificant-whitespace01': ('\n {% templatetag openblock %}\n', {}, '{%\n', + '\n {%\n'), + 'insignificant-whitespace02': ('\n{% templatetag openblock %}\n', {}, '{%\n', + '\n{%\n'), + 'insignificant-whitespace03': ('{% templatetag openblock %}\n', {}, '{%\n', + '{%\n'), + 'insignificant-whitespace04': ('\n\t \t {% templatetag openblock %}\n', {}, '{%\n', + '\n\t \t {%\n'), + # Leading whitespace with text before single template tag + 'insignificant-whitespace05': ('\n some\ttext {% templatetag openblock %}\n', {}, '\n some\ttext {%\n', + '\n some\ttext {%\n'), + # Leading line with text before single template tag + 'insignificant-whitespace06': ('\n some\ttext\n {% templatetag openblock %}\n', {}, '\n some\ttext{%\n', + '\n some\ttext\n {%\n'), + 'insignificant-whitespace07': ('\n some\ttext \n \t {% templatetag openblock %}\n', {}, '\n some\ttext {%\n', + '\n some\ttext \n \t {%\n'), + # whitespace leading /before/ the newline is not stripped. + 'insignificant-whitespace08': ('\n some\ttext \t \n {% templatetag openblock %}\n', {}, '\n some\ttext \t {%\n', + '\n some\ttext \t \n {%\n'), + # Multiple text lines before tag + 'insignificant-whitespace09': ('\n some\ntext \t \n {% templatetag openblock %}\n', {}, '\n some\ntext \t {%\n', + '\n some\ntext \t \n {%\n'), + 'insignificant-whitespace10': ('\n some \t \n \t text \t \n {% templatetag openblock %}\n', {}, '\n some \t \n \t text \t {%\n', + '\n some \t \n \t text \t \n {%\n'), + # Leading whitespace before tag, some text after + 'insignificant-whitespace11': ('\n \t {% templatetag openblock %} some text\n', {}, '\n \t {% some text\n', + '\n \t {% some text\n'), + # Leading whitespace before tag, some text with trailing whitespace after + 'insignificant-whitespace12': ('\n \t {% templatetag openblock %} some text \t \n', {}, '\n \t {% some text \t \n', + '\n \t {% some text \t \n'), + # Whitespace after tag is not removed + 'insignificant-whitespace13': ('\n \t {% templatetag openblock %} \t \n \t some text \t \n', {}, '{% \t \n \t some text \t \n', + '\n \t {% \t \n \t some text \t \n'), + # Multiple lines of leading whitespace. Only one leading newline is removed + 'insignificant-whitespace14': ('\n\n\n{% templatetag openblock %}\n some text\n', {}, '\n\n{%\n some text\n', + '\n\n\n{%\n some text\n'), + # Trailing whitespace after tag + 'insignificant-whitespace15': ('\n\n\n{% templatetag openblock %}\t \t \t\n some text\n', {}, '\n\n{%\t \t \t\n some text\n', + '\n\n\n{%\t \t \t\n some text\n'), + # Removable newline followed by leading whitespace + 'insignificant-whitespace16': ('\n\n\n\t \t \t{% templatetag openblock %}\n some text\n', {}, '\n\n{%\n some text\n', + '\n\n\n\t \t \t{%\n some text\n'), + # Removable leading whitespace and trailing whitespace + 'insignificant-whitespace17': ('\n\n\n\t \t \t{% templatetag openblock %}\t \t \t\n some text\n', {}, '\n\n{%\t \t \t\n some text\n', + '\n\n\n\t \t \t{%\t \t \t\n some text\n'), + # Multiple lines of trailing whitespace. No trailing newline is removed. + 'insignificant-whitespace18': ('\n{% templatetag openblock %}\n\n\n some text\n', {}, '{%\n\n\n some text\n', + '\n{%\n\n\n some text\n'), + 'insignificant-whitespace19': ('\n{% templatetag openblock %}\t \n\n\n some text\n', {}, '{%\t \n\n\n some text\n', + '\n{%\t \n\n\n some text\n'), + # Consecutive trimmed lines with tags strips one newline each + 'insignificant-whitespace20': ( + '\n{% templatetag openblock %}\n{% templatetag openblock %}\n{% templatetag openblock %}\n some text\n' + , {}, '{%{%{%\n some text\n', + '\n{%\n{%\n{%\n some text\n'), + # Consecutive trimmed lines with tags strips one newline each. Intermediate newlines are preserved + 'insignificant-whitespace21': ( + '\n\n{% templatetag openblock %}\n\n{% templatetag openblock %}\n\n{% templatetag openblock %}\n\n some text\n' + , {}, '\n{%\n{%\n{%\n\n some text\n', + '\n\n{%\n\n{%\n\n{%\n\n some text\n'), + # Consecutive trimmed lines with tags strips one newline each. Leading whitespace is stripped but trailing is not + 'insignificant-whitespace22': ( + '\n\n\t {% templatetag openblock %}\t \n\n\t {% templatetag openblock %}\t \n\n\t {% templatetag openblock %}\t \n some text\n' + , {}, '\n{%\t \n{%\t \n{%\t \n some text\n', + '\n\n\t {%\t \n\n\t {%\t \n\n\t {%\t \n some text\n'), + # Consecutive trimmed lines with tags strips one newline each. Intermediate whitespace is stripped + 'insignificant-whitespace23': ( + '\n\t {% templatetag openblock %}\t \n\t {% templatetag openblock %}\t \n\t {% templatetag openblock %}\t \n some text\n' + , {}, '{%\t {%\t {%\t \n some text\n', + '\n\t {%\t \n\t {%\t \n\t {%\t \n some text\n'), + # Intermediate whitespace on one line is preserved + # Consecutive tags on one line do not have intermediate whitespace or leading whitespace stripped + 'insignificant-whitespace24': ( + '\n\t {% templatetag openblock %}\t \t {% templatetag openblock %}\t \t {% templatetag openblock %}\t \n some text\n' + , {}, '\n\t {%\t \t {%\t \t {%\t \n some text\n', + '\n\t {%\t \t {%\t \t {%\t \n some text\n'), + # Still, only one leading newline is removed. + 'insignificant-whitespace25': ( + '\n\n {% templatetag openblock %}\n \t {% templatetag openblock %}\n \t {% templatetag openblock %}\n some text\n' + , {}, '\n{%{%{%\n some text\n', + '\n\n {%\n \t {%\n \t {%\n some text\n'), + # Lines with {# comments #} have the same stripping behavior + 'insignificant-whitespace26': ( + '\n\n {% templatetag openblock %}\n \t {# some comment #}\n some text\n' + , {}, '\n{%\n some text\n', + '\n\n {%\n \t \n some text\n'), + # Only {# comments #} + 'insignificant-whitespace27': ( + '\n\n {# a comment #}\n \t {# some comment #}\n some text\n' + , {}, '\n\n some text\n', + '\n\n \n \t \n some text\n'), + # Consecutive newlines with tags and comments + 'insignificant-whitespace28': ( + '\n\t {% templatetag openblock %}\t \n\t {# some comment #}\t \n\t {% templatetag openblock %}\t \n some text\n' + , {}, '{%\t \t {%\t \n some text\n', + '\n\t {%\t \n\t \t \n\t {%\t \n some text\n'), + + # Lines with only {{ values }} have the same stripping behavior + 'insignificant-whitespace29': ( + '\n {% templatetag openblock %}\t\n \t {{ spam }}\t \n \t {% templatetag openblock %}\t \n some text\n' + , {"spam" : "ham"}, '{%\tham\t {%\t \n some text\n', + '\n {%\t\n \t ham\t \n \t {%\t \n some text\n'), + 'insignificant-whitespace30': ( + '\n\n {% templatetag openblock %}\t\n\n \t {{ spam }}\t \n\n \t {% templatetag openblock %}\t \n some text\n' + , {"spam" : "ham"}, '\n{%\t\nham\t \n{%\t \n some text\n', + '\n\n {%\t\n\n \t ham\t \n\n \t {%\t \n some text\n'), + ## Leading whitespace not stripped when followed by anything. See insignificant-whitespace24 + 'insignificant-whitespace31': ( + '\n {% templatetag openblock %}\t \t {{ spam }}\t \t {% templatetag openblock %}\t \n some text\n' + , {"spam" : "ham"}, '\n {%\t \t ham\t \t {%\t \n some text\n', + '\n {%\t \t ham\t \t {%\t \n some text\n'), + # {{ value }} {% tag %} {{ value }} this time + 'insignificant-whitespace32': ( + '\n {{ spam }}\t\n \t {% templatetag openblock %}\t \n \t {{ spam }}\t \n some text\n' + , {"spam" : "ham"}, 'ham\t{%\t ham\t \n some text\n', + '\n ham\t\n \t {%\t \n \t ham\t \n some text\n'), + + # Invalid stuff is still invalid + # Newlines inside begin-end tokens, even in {# comments #}, make it not a tag. + 'insignificant-whitespace33': ( + '\n\n {# \n{% templatetag openblock #}\t \n some text\n' + , {}, '\n\n {# \n{% templatetag openblock #}\t \n some text\n', + '\n\n {# \n{% templatetag openblock #}\t \n some text\n'), + # Complete comment matching tags on one line are processed + 'insignificant-whitespace34': ( + '\n\n {# \n{# templatetag openblock #}\t \n some text\n' + , {}, '\n\n {# \t \n some text\n', + '\n\n {# \n\t \n some text\n'), + 'insignificant-whitespace35': ( + '\n\n {# \n{# templatetag openblock\n #}\t \n some text\n' + , {}, '\n\n {# \n{# templatetag openblock\n #}\t \n some text\n', + '\n\n {# \n{# templatetag openblock\n #}\t \n some text\n'), + 'insignificant-whitespace36': ( + '\n\n {# \n{{ some comment #}\t \n some text\n' + , {}, '\n\n {# \n{{ some comment #}\t \n some text\n', + '\n\n {# \n{{ some comment #}\t \n some text\n'), + 'insignificant-whitespace37': ( + '\n\n {# \n \t {% templatetag openblock #}\t \n some text\n' + , {}, '\n\n {# \n \t {% templatetag openblock #}\t \n some text\n', + '\n\n {# \n \t {% templatetag openblock #}\t \n some text\n'), + 'insignificant-whitespace38': ( + "\n\n {# templatetag openblock #\n}\t \n some text\n" + , {}, "\n\n {# templatetag openblock #\n}\t \n some text\n", + "\n\n {# templatetag openblock #\n}\t \n some text\n" ), + 'insignificant-whitespace39': ( + "\n\n {% templatetag openblock %\n}\t \n some text\n" + , {}, "\n\n {% templatetag openblock %\n}\t \n some text\n", + "\n\n {% templatetag openblock %\n}\t \n some text\n" ), + 'insignificant-whitespace40': ( + "\n\n {{ templatetag openblock }\n}\t \n some text\n" + , {}, "\n\n {{ templatetag openblock }\n}\t \n some text\n", + "\n\n {{ templatetag openblock }\n}\t \n some text\n" ), + 'insignificant-whitespace41': ( + "\n\n {\n# {# templatetag openblock #}\t \n some text\n" + , {}, "\n\n {\n# \t \n some text\n", + "\n\n {\n# \t \n some text\n"), + 'insignificant-whitespace42': ( + "\n\n {\n {# templatetag openblock #}\t \n some text\n" + , {}, "\n\n {\t \n some text\n", + "\n\n {\n \t \n some text\n"), + 'insignificant-whitespace43': ( + "\n{{# foo #};{# bar #}\n" + , {}, "\n{;\n", + "\n{;\n"), + } + tests = whitespace_tests.items() + tests.sort() + + # Register our custom template loader. + def test_whitespace_loader(template_name, template_dirs=None): + "A custom template loader that loads the unit-test templates." + try: + return (whitespace_tests[template_name][0] , "test:%s" % template_name) + except KeyError: + raise template.TemplateDoesNotExist, template_name + + old_template_loaders = loader.template_source_loaders + loader.template_source_loaders = [test_whitespace_loader] + + failures = [] + + old_strip_leading_whitespace = settings.TEMPLATE_STRIP_LEADING_WHITESPACE + + for name, vals in tests: + for strip_leading_whitespace in (True, False): + settings.TEMPLATE_STRIP_LEADING_WHITESPACE = strip_leading_whitespace + test_template = loader.get_template(name) + result = vals[2] if strip_leading_whitespace else vals[3] + output = self.render(test_template, vals) + + if output != result: + failures.append("Whitespace test: %s -- FAILED. Expected %r, got %r" % (name, result, output)) + + loader.template_source_loaders = old_template_loaders + settings.TEMPLATE_STRIP_LEADING_WHITESPACE = old_strip_leading_whitespace + + self.assertEqual(failures, [], "Tests failed:\n%s\n%s" % + ('-'*70, ("\n%s\n" % ('-'*70)).join(failures))) + def test_templates(self): template_tests = self.get_template_tests() filter_tests = filters.get_filter_tests()