Index: django/core/template_loader.py =================================================================== --- django/core/template_loader.py (revision 854) +++ django/core/template_loader.py (working copy) @@ -1,6 +1,6 @@ "Wrapper for loading templates from storage of some sort (e.g. files or db)" import template -from template_file import load_template_source +from template_file import find_template_source class ExtendsError(Exception): pass @@ -10,14 +10,14 @@ Returns a compiled template.Template object for the given template name, handling template inheritance recursively. """ - return get_template_from_string(load_template_source(template_name)) + return get_template_from_string(*find_template_source(template_name)) -def get_template_from_string(source): +def get_template_from_string(source, filename=template.UNKNOWN_SOURCE): """ Returns a compiled template.Template object for the given template code, handling template inheritance recursively. """ - return template.Template(source) + return template.Template(source, filename) def render_to_string(template_name, dictionary=None, context_instance=None): """ @@ -90,7 +90,7 @@ error_msg += " Got this from the %r variable." % self.parent_name_var raise template.TemplateSyntaxError, error_msg try: - return get_template_from_string(load_template_source(parent, self.template_dirs)) + return get_template_from_string(*find_template_source(parent, self.template_dirs)) except template.TemplateDoesNotExist: raise template.TemplateSyntaxError, "Template %r cannot be extended, because it doesn't exist" % parent @@ -142,7 +142,7 @@ This tag may be used in two ways: ``{% extends "base" %}`` (with quotes) uses the literal value "base" as the name of the parent template to extend, - or ``{% entends variable %}`` uses the value of ``variable`` as the name + or ``{% extends variable %}`` uses the value of ``variable`` as the name of the parent template to extend. """ bits = token.contents.split() Index: django/core/template_file.py =================================================================== --- django/core/template_file.py (revision 854) +++ django/core/template_file.py (working copy) @@ -4,14 +4,16 @@ from django.core.template import TemplateDoesNotExist import os -def load_template_source(template_name, template_dirs=None): + +def find_template_source(template_name, template_dirs=None): + "Returns a tuple of (template_string, filepath)." if not template_dirs: template_dirs = TEMPLATE_DIRS tried = [] for template_dir in template_dirs: filepath = os.path.join(template_dir, template_name) + TEMPLATE_FILE_EXTENSION try: - return open(filepath).read() + return (open(filepath).read(), filepath) except IOError: tried.append(filepath) if template_dirs: @@ -19,3 +21,7 @@ else: error_msg = "Your TEMPLATE_DIRS settings is empty. Change it to point to at least one template directory." raise TemplateDoesNotExist, error_msg + + +def load_template_source(template_name, template_dirs=None): + return find_template_source(template_name, template_dirs)[0] Index: django/core/template.py =================================================================== --- django/core/template.py (revision 854) +++ django/core/template.py (working copy) @@ -55,7 +55,7 @@ '\n\n\n\n' """ import re -from django.conf.settings import DEFAULT_CHARSET +from django.conf.settings import DEFAULT_CHARSET, DEBUG __all__ = ('Template','Context','compile_string') @@ -74,6 +74,12 @@ ALLOWED_VARIABLE_CHARS = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_.' +#What to report as the origin of templates that come from non file sources (eg strings) +UNKNOWN_SOURCE="" + +#match starts of lines +newline_re = re.compile("^", re.M); + # match a variable or block tag and capture the entire tag, including start/end delimiters tag_re = re.compile('(%s.*?%s|%s.*?%s)' % (re.escape(BLOCK_TAG_START), re.escape(BLOCK_TAG_END), re.escape(VARIABLE_TAG_START), re.escape(VARIABLE_TAG_END))) @@ -102,10 +108,10 @@ pass class Template: - def __init__(self, template_string): + def __init__(self, template_string, filename=UNKNOWN_SOURCE): "Compilation stage" - self.nodelist = compile_string(template_string) - + self.nodelist = compile_string(template_string, filename) + def __iter__(self): for node in self.nodelist: for subnode in node: @@ -115,10 +121,17 @@ "Display stage -- can be called many times" return self.nodelist.render(context) -def compile_string(template_string): +def compile_string(template_string, filename): "Compiles template_string into NodeList ready for rendering" - tokens = tokenize(template_string) - parser = Parser(tokens) + if DEBUG: + lexer_factory = DebugLexer + parser_factory = DebugParser + else: + lexer_factory = Lexer + parser_factory = Parser + + lexer = lexer_factory(template_string, filename) + parser = parser_factory(lexer.tokenize()) return parser.parse() class Context: @@ -178,22 +191,77 @@ self.contents[:20].replace('\n', '') ) -def tokenize(template_string): - "Return a list of tokens from a given template_string" - # remove all empty strings, because the regex has a tendency to add them - bits = filter(None, tag_re.split(template_string)) - return map(create_token, bits) +class Lexer(object): + def __init__(self, template_string, filename): + self.template_string = template_string + self.filename = filename + + def tokenize(self): + "Return a list of tokens from a given template_string" + bits = filter(None, tag_re.split(self.template_string)) + return map(self.create_token, bits) + + def create_token(self,token_string): + "Convert the given token string into a new Token object and return it" + if token_string.startswith(VARIABLE_TAG_START): + token = Token(TOKEN_VAR, token_string[len(VARIABLE_TAG_START):-len(VARIABLE_TAG_END)].strip()) + elif token_string.startswith(BLOCK_TAG_START): + token = Token(TOKEN_BLOCK, token_string[len(BLOCK_TAG_START):-len(BLOCK_TAG_END)].strip()) + else: + token = Token(TOKEN_TEXT, token_string) + return token -def create_token(token_string): - "Convert the given token string into a new Token object and return it" - if token_string.startswith(VARIABLE_TAG_START): - return Token(TOKEN_VAR, token_string[len(VARIABLE_TAG_START):-len(VARIABLE_TAG_END)].strip()) - elif token_string.startswith(BLOCK_TAG_START): - return Token(TOKEN_BLOCK, token_string[len(BLOCK_TAG_START):-len(BLOCK_TAG_END)].strip()) - else: - return Token(TOKEN_TEXT, token_string) +class DebugLexer(Lexer): + def __init__(self, template_string, filename): + super(DebugLexer,self).__init__(template_string, filename) -class Parser: + def find_linebreaks(self, template_string): + for match in newline_re.finditer(template_string): + yield match.start() + + def tokenize(self): + "Return a list of tokens from a given template_string" + + token_tups = [] + upto = 0 + line = 1 + #TODO:Py2.4 generator expression + linebreaks = self.find_linebreaks(self.template_string) + next_linebreak = linebreaks.next() + try: + for match in tag_re.finditer(self.template_string): + start, end = match.span() + if start > upto: + token_tups.append( (self.template_string[upto:start], line) ) + upto = start + + while next_linebreak <= upto: + next_linebreak = linebreaks.next() + line += 1 + + token_tups.append( (self.template_string[start:end], line) ) + upto = end + + while next_linebreak <= upto: + next_linebreak = linebreaks.next() + line += 1 + except StopIteration: + pass + + last_bit = self.template_string[upto:] + if len(last_bit): + token_tups.append( (last_bit, line) ) + + return [ self.create_token(tok, (self.filename, line)) for tok, line in token_tups] + + + def create_token(self, token_string, source): + token = super(DebugLexer, self).create_token(token_string) + token.source = source + return token + + +class Parser(object): def __init__(self, tokens): self.tokens = tokens @@ -202,11 +270,11 @@ while self.tokens: token = self.next_token() if token.token_type == TOKEN_TEXT: - nodelist.append(TextNode(token.contents)) + self.extend_nodelist(nodelist, TextNode(token.contents), token) elif token.token_type == TOKEN_VAR: if not token.contents: - raise TemplateSyntaxError, "Empty variable tag" - nodelist.append(VariableNode(token.contents)) + self.empty_variable(token) + self.extend_nodelist(nodelist, VariableNode(token.contents), token) elif token.token_type == TOKEN_BLOCK: if token.contents in parse_until: # put token back on token list so calling code knows why it terminated @@ -215,16 +283,44 @@ try: command = token.contents.split()[0] except IndexError: - raise TemplateSyntaxError, "Empty block tag" + self.empty_block_tag(token) + + # execute callback function for this tag and append resulting node + self.enter_command(command, token); try: - # execute callback function for this tag and append resulting node - nodelist.append(registered_tags[command](self, token)) + compile_func = registered_tags[command] except KeyError: - raise TemplateSyntaxError, "Invalid block tag: '%s'" % command + self.invalid_block_tag(token, command) + + self.extend_nodelist(nodelist, compile_func(self, token), token) + self.exit_command(); + if parse_until: - raise TemplateSyntaxError, "Unclosed tag(s): '%s'" % ', '.join(parse_until) + self.unclosed_block_tag(token) + return nodelist + def extend_nodelist(self, nodelist, node, token): + nodelist.append(node) + + def enter_command(self, command, token): + pass + + def exit_command(self): + pass + + def empty_variable(self, token): + raise TemplateSyntaxError, "Empty variable tag" + + def empty_block_tag(self, token): + raise TemplateSyntaxError, "Empty block tag" + + def invalid_block_tag(self, token, command): + raise TemplateSyntaxError, "Invalid block tag: %s" % (command) + + def unclosed_block_tag(self, token): + raise TemplateSyntaxError, "Unclosed tags: %s " % ', '.join(parse_until) + def next_token(self): return self.tokens.pop(0) @@ -234,6 +330,40 @@ def delete_first_token(self): del self.tokens[0] + +class DebugParser(Parser): + def __init__(self, lexer): + super(DebugParser, self).__init__(lexer) + self.command_stack = [] + + def enter_command(self, command, token): + self.command_stack.append( (command, token.source) ) + + def exit_command(self): + self.command_stack.pop() + + def format_source(self, source): + return "at %s, line %d" % source + + def extend_nodelist(self, nodelist, node, token): + node.source = token.source + super(DebugParser, self).extend_nodelist(nodelist, node, token) + + def empty_variable(self, token): + raise TemplateSyntaxError, "Empty variable tag %s" % self.format_source(token.source) + + def empty_block_tag(self, token): + raise TemplateSyntaxError, "Empty block tag %s" % self.format_source(token.source) + + def invalid_block_tag(self, token, command): + raise TemplateSyntaxError, "Invalid block tag: '%s' %s" % (command, self.format_source(token.source)) + + def unclosed_block_tag(self, token): + (command, (file,line)) = self.command_stack.pop() + msg = "Unclosed tag '%s' starting at %s, line %d. Looking for one of: %s " % \ + (command, file, line, ', '.join(parse_until) ) + raise TemplateSyntaxError, msg + class FilterParser: """Parse a variable token and its optional filters (all as a single string), and return a list of tuples of the filter name and arguments.