Ticket #2594: template_whitespace_2.patch

django/template/__init__.py

@@ -67 +67 @@
 TOKEN_VAR = 1
 TOKEN_BLOCK = 2
 TOKEN_COMMENT = 3
+TOKEN_WHITESPACE = 4
+TOKEN_MULTIPART = 5
 
 # template syntax constants
 FILTER_SEPARATOR = '|'
@@ -87 +89 @@
 # (e.g. strings)
 UNKNOWN_SOURCE="<unknown source>"
 
-# match a variable or block tag and capture the entire tag, including start/end delimiters
-tag_re = re.compile('(%s.*?%s|%s.*?%s|%s.*?%s)' % (re.escape(BLOCK_TAG_START), re.escape(BLOCK_TAG_END),
+BLOCKS_RE = r'%s.*?%s|%s.*?%s|%s.*?%s' % (re.escape(BLOCK_TAG_START), re.escape(BLOCK_TAG_END),
                                           re.escape(VARIABLE_TAG_START), re.escape(VARIABLE_TAG_END),
-                                          re.escape(COMMENT_TAG_START), re.escape(COMMENT_TAG_END)))
+                                          re.escape(COMMENT_TAG_START), re.escape(COMMENT_TAG_END))
+# Match a variable or block tag and capture the entire tag, including tag
+# delimiters (and surrounding white space if the tag is on its own line).
+tag_re = re.compile('((?<![^\n])[\t ]*(?:%s)[\t ]*\n|(?:%s))' % (BLOCKS_RE, BLOCKS_RE))
+whitespace_tag_re = re.compile('(%s)' % BLOCKS_RE)
 
 # global dictionary of libraries that have been loaded using get_library
 libraries = {}
@@ -167 +172 @@
 
 class Token(object):
     def __init__(self, token_type, contents):
-        "The token_type must be TOKEN_TEXT, TOKEN_VAR, TOKEN_BLOCK or TOKEN_COMMENT"
+        """
+        The token_type must be one of the following: TOKEN_TEXT, TOKEN_VAR,
+        TOKEN_BLOCK, TOKEN_COMMENT, TOKEN_WHITESPACE, TOKEN_MULTIPART.
+        """
         self.token_type, self.contents = token_type, contents
 
     def __str__(self):
-        return '<%s token: "%s...">' % \
-            ({TOKEN_TEXT: 'Text', TOKEN_VAR: 'Var', TOKEN_BLOCK: 'Block', TOKEN_COMMENT: 'Comment'}[self.token_type],
-            self.contents[:20].replace('\n', ''))
+        if self.token_type == TOKEN_MULTIPART:
+            contents = ', '.join([str(token) for token in self.contents])
+        elif self.token_type == TOKEN_WHITESPACE:
+            contents = self.contents.replace('\n', '\\n')
+        else:
+            contents = self.contents.replace('\n', '')
+        if len(self.contents) > 23:
+            contents = contents[:20] + '...'
+        return '<%s token: "%s">' % \
+            ({TOKEN_TEXT: 'Text',
+              TOKEN_VAR: 'Var',
+              TOKEN_BLOCK: 'Block',
+              TOKEN_COMMENT: 'Comment',
+              TOKEN_WHITESPACE: 'Whitespace',
+              TOKEN_MULTIPART: 'Multipart'}[self.token_type],
+            contents)
 
     def split_contents(self):
         return list(smart_split(self.contents))
@@ -191 +212 @@
 
     def create_token(self,token_string):
         "Convert the given token string into a new Token object and return it"
-        if token_string.startswith(VARIABLE_TAG_START):
+        if token_string.endswith('\n'):
+            token_group = whitespace_tag_re.split(token_string)
+            has_whitespace = len(token_group) > 1
+        else:
+            has_whitespace = False
+        if has_whitespace:
+            token = self.multipart_token(token_group)
+        elif token_string.startswith(VARIABLE_TAG_START):
             token = Token(TOKEN_VAR, token_string[len(VARIABLE_TAG_START):-len(VARIABLE_TAG_END)].strip())
         elif token_string.startswith(BLOCK_TAG_START):
             token = Token(TOKEN_BLOCK, token_string[len(BLOCK_TAG_START):-len(BLOCK_TAG_END)].strip())
@@ -201 +229 @@
             token = Token(TOKEN_TEXT, token_string)
         return token
 
+    def multipart_token(self, token_group):
+        before = Token(TOKEN_WHITESPACE, token_group[0])
+        before.start = True
+        tag = self.create_token(token_group[1])
+        after = Token(TOKEN_WHITESPACE, token_group[2])
+        after.start = False
+        return Token(TOKEN_MULTIPART, (before, tag, after))
+
 class DebugLexer(Lexer):
     def __init__(self, template_string, origin):
         super(DebugLexer, self).__init__(template_string, origin)
@@ -220 +256 @@
             token_tups.append( (last_bit, (upto, upto + len(last_bit))) )
         return [self.create_token(tok, (self.origin, loc)) for tok, loc in token_tups]
 
-    def create_token(self, token_string, source):
+    def create_token(self, token_string, source=''):
         token = super(DebugLexer, self).create_token(token_string)
+        if token.token_type == TOKEN_MULTIPART:
+            for sub_token in token.contents:
+                sub_token.source = source
         token.source = source
         return token
 
@@ -238 +277 @@
         nodelist = self.create_nodelist()
         while self.tokens:
             token = self.next_token()
-            if token.token_type == TOKEN_TEXT:
+            if token.token_type == TOKEN_WHITESPACE:
+                if token.start:
+                    node_type = WhitespaceStartNode
+                else:
+                    node_type = WhitespaceEndNode
+                self.extend_nodelist(nodelist, node_type(token.contents), token)
+            elif token.token_type == TOKEN_TEXT:
                 self.extend_nodelist(nodelist, TextNode(token.contents), token)
             elif token.token_type == TOKEN_VAR:
                 if not token.contents:
@@ -313 +358 @@
         pass
 
     def next_token(self):
-        return self.tokens.pop(0)
+        token = self.tokens.pop(0)
+        if token.token_type == TOKEN_MULTIPART:
+            token_group = list(token.contents)
+            # Since we are prepending, we want the last token in first.
+            token_group.reverse()
+            map(self.prepend_token, token_group)
+            token = self.next_token()
+        return token
 
     def prepend_token(self, token):
         self.tokens.insert(0, token)
 
     def delete_first_token(self):
-        del self.tokens[0]
+        self.next_token()
 
     def add_library(self, lib):
         self.tags.update(lib.tags)
@@ -689 +741 @@
 class NodeList(list):
     def render(self, context):
         bits = []
+        whitespace = None
+        whitespace_needed = False
+        open_whitespace_block = False
         for node in self:
             if isinstance(node, Node):
-                bits.append(self.render_node(node, context))
+                bit = self.render_node(node, context)
             else:
-                bits.append(node)
+                bit = node
+            if isinstance(node, WhitespaceStartNode):
+                # Remember the starting white space, but don't use it yet.
+                whitespace = bit
+                bit = None
+                whitespace_start_position = len(bits)
+                open_whitespace_block = True
+                whitespace_needed = False
+            elif isinstance(node, WhitespaceEndNode):
+                if not whitespace_needed:
+                    # If there is stored starting white space and this is the
+                    # end of the line, then the white space and this new line
+                    # can be dropped.
+                    open_whitespace_block = False
+                elif bits[-1].endswith('\n'):
+                    # If the content ends in a new line, don't use the
+                    # surrounding white space.
+                    open_whitespace_block = False
+                if open_whitespace_block:
+                    # Insert the starting white space.
+                    if whitespace:
+                        bits.insert(whitespace_start_position, whitespace)
+                    open_whitespace_block = False
+                else:
+                    # Drop the new line.
+                    bit = None
+            if bit:
+                bits.append(bit)
+                if open_whitespace_block and not whitespace_needed:
+                    whitespace_needed = True
         return ''.join(bits)
 
     def get_nodes_by_type(self, nodetype):
@@ -762 +846 @@
             raise
         return self.encode_output(output)
 
+class WhitespaceStartNode(TextNode):
+    def __repr__(self):
+        return "<Whitespace Start Node: %r>" % self.s[:25]
+
+class WhitespaceEndNode(TextNode):
+    def __repr__(self):
+        return "<Whitespace End Node: %r>" % self.s[:25]
+
 def generic_tag_compiler(params, defaults, name, node_class, parser, token):
     "Returns a template.Node subclass."
     bits = token.split_contents()[1:]

django/template/defaulttags.py

@@ -118 +118 @@
                 'parentloop': parentloop,
             }
             context[self.loopvar] = item
-            for node in self.nodelist_loop:
-                nodelist.append(node.render(context))
+            nodelist.extend(self.nodelist_loop.render(context))
         context.pop()
         return nodelist.render(context)
 

django/template/loader_tags.py

@@ -59 +59 @@
         else:
             return get_template_from_string(source, origin, parent)
 
+    def get_parent_extendsnode(self, compiled_parent):
+        """
+        Returns the ExtendsNode for the parent template.
+        None is returned if a match is not found in the first 10 nodes.
+        """
+        for node in compiled_parent.nodelist[:10]:
+            if isinstance(node, ExtendsNode):
+                return node
+        return None
+
     def render(self, context):
         compiled_parent = self.get_parent(context)
-        parent_is_child = isinstance(compiled_parent.nodelist[0], ExtendsNode)
+        parent_extendsnode = self.get_parent_extendsnode(compiled_parent)
         parent_blocks = dict([(n.name, n) for n in compiled_parent.nodelist.get_nodes_by_type(BlockNode)])
         for block_node in self.nodelist.get_nodes_by_type(BlockNode):
             # Check for a BlockNode with this node's name, and replace it if found.
@@ -72 +82 @@
                 # parent block might be defined in the parent's *parent*, so we
                 # add this BlockNode to the parent's ExtendsNode nodelist, so
                 # it'll be checked when the parent node's render() is called.
-                if parent_is_child:
-                    compiled_parent.nodelist[0].nodelist.append(block_node)
+                if parent_extendsnode:
+                    parent_extendsnode.nodelist.append(block_node)
             else:
                 # Keep any existing parents and add a new one. Used by BlockNode.
                 parent_block.parent = block_node.parent