Ticket #603: django-template-errors-4.patch

File django-template-errors-4.patch, 16.4 KB (added by anonymous, 19 years ago)

updated for apploaders and got rid of exceptions in normal path of debug lexer.

  • django/core/template/__init__.py

     
    5555'\n<html>\n\n</html>\n'
    5656"""
    5757import re
    58 from django.conf.settings import DEFAULT_CHARSET
     58from django.conf.settings import DEFAULT_CHARSET, DEBUG
    5959
    6060__all__ = ('Template','Context','compile_string')
    6161
    … …  
    7474
    7575ALLOWED_VARIABLE_CHARS = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_.'
    7676
     77#What to report as the origin of templates that come from non file sources (eg strings)
     78UNKNOWN_SOURCE="<unknown source>"
     79
     80#match starts of lines
     81newline_re = re.compile("^", re.M);
     82
    7783# match a variable or block tag and capture the entire tag, including start/end delimiters
    7884tag_re = re.compile('(%s.*?%s|%s.*?%s)' % (re.escape(BLOCK_TAG_START), re.escape(BLOCK_TAG_END),
    7985                                          re.escape(VARIABLE_TAG_START), re.escape(VARIABLE_TAG_END)))
    … …  
    102108    pass
    103109
    104110class Template:
    105     def __init__(self, template_string):
     111    def __init__(self, template_string, filename=UNKNOWN_SOURCE):
    106112        "Compilation stage"
    107         self.nodelist = compile_string(template_string)
    108 
     113        self.nodelist = compile_string(template_string, filename)
     114       
    109115    def __iter__(self):
    110116        for node in self.nodelist:
    111117            for subnode in node:
    … …  
    115121        "Display stage -- can be called many times"
    116122        return self.nodelist.render(context)
    117123
    118 def compile_string(template_string):
     124def compile_string(template_string, filename):
    119125    "Compiles template_string into NodeList ready for rendering"
    120     tokens = tokenize(template_string)
    121     parser = Parser(tokens)
     126    if DEBUG:
     127        lexer_factory = DebugLexer
     128        parser_factory = DebugParser
     129    else:
     130        lexer_factory = Lexer
     131        parser_factory = Parser
     132       
     133    lexer = lexer_factory(template_string, filename)
     134    parser = parser_factory(lexer.tokenize())
    122135    return parser.parse()
    123136
    124137class Context:
    … …  
    177190            {TOKEN_TEXT:'Text', TOKEN_VAR:'Var', TOKEN_BLOCK:'Block'}[self.token_type],
    178191            self.contents[:20].replace('\n', '')
    179192            )
     193           
     194    def __repr__(self):
     195        return '<%s token: "%s">' % (
     196            {TOKEN_TEXT:'Text', TOKEN_VAR:'Var', TOKEN_BLOCK:'Block'}[self.token_type],
     197            self.contents[:].replace('\n', '')
     198            )
    180199
    181 def tokenize(template_string):
    182     "Return a list of tokens from a given template_string"
    183     # remove all empty strings, because the regex has a tendency to add them
    184     bits = filter(None, tag_re.split(template_string))
    185     return map(create_token, bits)
     200class Lexer(object):
     201    def __init__(self, template_string, filename):
     202        self.template_string = template_string
     203        self.filename = filename
     204   
     205    def tokenize(self):
     206        "Return a list of tokens from a given template_string"
     207        bits = filter(None, tag_re.split(self.template_string))
     208        return map(self.create_token, bits)
     209       
     210    def create_token(self,token_string):
     211        "Convert the given token string into a new Token object and return it"
     212        if token_string.startswith(VARIABLE_TAG_START):
     213            token = Token(TOKEN_VAR, token_string[len(VARIABLE_TAG_START):-len(VARIABLE_TAG_END)].strip())
     214        elif token_string.startswith(BLOCK_TAG_START):
     215            token = Token(TOKEN_BLOCK, token_string[len(BLOCK_TAG_START):-len(BLOCK_TAG_END)].strip())
     216        else:
     217            token = Token(TOKEN_TEXT, token_string)
     218        return token
    186219
    187 def create_token(token_string):
    188     "Convert the given token string into a new Token object and return it"
    189     if token_string.startswith(VARIABLE_TAG_START):
    190         return Token(TOKEN_VAR, token_string[len(VARIABLE_TAG_START):-len(VARIABLE_TAG_END)].strip())
    191     elif token_string.startswith(BLOCK_TAG_START):
    192         return Token(TOKEN_BLOCK, token_string[len(BLOCK_TAG_START):-len(BLOCK_TAG_END)].strip())
    193     else:
    194         return Token(TOKEN_TEXT, token_string)
     220class DebugLexer(Lexer):
     221    def __init__(self, template_string, filename):
     222        super(DebugLexer,self).__init__(template_string, filename)
    195223
    196 class Parser:
     224    def find_linebreaks(self, template_string):
     225        for match in newline_re.finditer(template_string):
     226            yield match.start()
     227        yield len(template_string) + 1
     228
     229    def tokenize(self):
     230        "Return a list of tokens from a given template_string"
     231        token_tups, upto = [], 0
     232        lines = enumerate(self.find_linebreaks(self.template_string))
     233        line, next_linebreak = lines.next()
     234        for match in tag_re.finditer(self.template_string):
     235            while next_linebreak <= upto:
     236                line, next_linebreak = lines.next()   
     237            start, end = match.span()
     238            if start > upto:       
     239                token_tups.append( (self.template_string[upto:start], line) )
     240                upto = start
     241                while next_linebreak <= upto:
     242                    line, next_linebreak = lines.next()             
     243            token_tups.append( (self.template_string[start:end], line) )
     244            upto = end
     245        last_bit = self.template_string[upto:]
     246        if last_bit:
     247           token_tups.append( (last_bit, line) )
     248        return [ self.create_token(tok, (self.filename, line)) for tok, line in token_tups]
     249
     250    def create_token(self, token_string, source):
     251        token = super(DebugLexer, self).create_token(token_string)
     252        token.source = source
     253        return token
     254
     255class Parser(object):
    197256    def __init__(self, tokens):
    198257        self.tokens = tokens
     258       
    199259
    200260    def parse(self, parse_until=[]):
    201261        nodelist = NodeList()
    202262        while self.tokens:
    203263            token = self.next_token()
    204264            if token.token_type == TOKEN_TEXT:
    205                 nodelist.append(TextNode(token.contents))
     265                self.extend_nodelist(nodelist, TextNode(token.contents), token)
    206266            elif token.token_type == TOKEN_VAR:
    207267                if not token.contents:
    208                     raise TemplateSyntaxError, "Empty variable tag"
    209                 nodelist.append(VariableNode(token.contents))
     268                    self.empty_variable(token)
     269                self.extend_nodelist(nodelist, VariableNode(token.contents), token)
    210270            elif token.token_type == TOKEN_BLOCK:
    211271                if token.contents in parse_until:
    212272                    # put token back on token list so calling code knows why it terminated
    … …  
    215275                try:
    216276                    command = token.contents.split()[0]
    217277                except IndexError:
    218                     raise TemplateSyntaxError, "Empty block tag"
     278                    self.empty_block_tag(token)
     279               
     280                # execute callback function for this tag and append resulting node
     281                self.enter_command(command, token);
    219282                try:
    220                     # execute callback function for this tag and append resulting node
    221                     nodelist.append(registered_tags[command](self, token))
     283                    compile_func = registered_tags[command]
    222284                except KeyError:
    223                     raise TemplateSyntaxError, "Invalid block tag: '%s'" % command
     285                    self.invalid_block_tag(token, command)
     286                   
     287                self.extend_nodelist(nodelist, compile_func(self, token), token)
     288                self.exit_command();
     289               
    224290        if parse_until:
    225             raise TemplateSyntaxError, "Unclosed tag(s): '%s'" % ', '.join(parse_until)
     291            self.unclosed_block_tag(token, parse_until)
     292           
    226293        return nodelist
    227294
     295    def extend_nodelist(self, nodelist, node, token):
     296        nodelist.append(node)
     297
     298    def enter_command(self, command, token):
     299        pass
     300       
     301    def exit_command(self):
     302        pass
     303
     304    def empty_variable(self, token):
     305        raise TemplateSyntaxError, "Empty variable tag"
     306   
     307    def empty_block_tag(self, token):
     308        raise TemplateSyntaxError, "Empty block tag"
     309   
     310    def invalid_block_tag(self, token, command):
     311        raise TemplateSyntaxError, "Invalid block tag: %s" % (command)
     312   
     313    def unclosed_block_tag(self, token, parse_until):
     314        raise TemplateSyntaxError, "Unclosed tags: %s " %  ', '.join(parse_until)
     315       
    228316    def next_token(self):
    229317        return self.tokens.pop(0)
    230318
    … …  
    234322    def delete_first_token(self):
    235323        del self.tokens[0]
    236324
     325
     326class DebugParser(Parser):
     327    def __init__(self, lexer):
     328        super(DebugParser, self).__init__(lexer)
     329        self.command_stack = []
     330
     331    def enter_command(self, command, token):
     332        self.command_stack.append( (command, token.source) )
     333       
     334    def exit_command(self):
     335        self.command_stack.pop()
     336
     337    def format_source(self, source):
     338        return "at %s, line %d" % source
     339
     340    def extend_nodelist(self, nodelist, node, token):
     341        node.source = token.source
     342        super(DebugParser, self).extend_nodelist(nodelist, node, token)
     343
     344    def empty_variable(self, token):
     345        raise TemplateSyntaxError, "Empty variable tag %s" % self.format_source(token.source)
     346   
     347    def empty_block_tag(self, token):
     348        raise TemplateSyntaxError, "Empty block tag %s" % self.format_source(token.source)
     349   
     350    def invalid_block_tag(self, token, command):
     351        raise TemplateSyntaxError, "Invalid block tag: '%s' %s" % (command, self.format_source(token.source))
     352   
     353    def unclosed_block_tag(self, token, parse_until):
     354        (command, (file,line)) = self.command_stack.pop()
     355        msg = "Unclosed tag '%s' starting at %s, line %d. Looking for one of: %s " % \
     356              (command, file, line, ', '.join(parse_until) )
     357        raise TemplateSyntaxError, msg
     358
    237359class FilterParser:
    238360    """Parse a variable token and its optional filters (all as a single string),
    239361       and return a list of tuples of the filter name and arguments.

  • django/core/template/loaders/app_directories.py

     
    2121    for template_dir in app_template_dirs:
    2222        filepath = os.path.join(template_dir, template_name) + TEMPLATE_FILE_EXTENSION
    2323        try:
    24             return open(filepath).read()
     24            return (open(filepath).read(), filepath)
    2525        except IOError:
    2626            pass
    2727    raise TemplateDoesNotExist, template_name

  • django/core/template/loaders/filesystem.py

     
    1111    for template_dir in template_dirs:
    1212        filepath = os.path.join(template_dir, template_name) + TEMPLATE_FILE_EXTENSION
    1313        try:
    14             return open(filepath).read()
     14            return (open(filepath).read(), filepath)
    1515        except IOError:
    1616            tried.append(filepath)
    1717    if template_dirs:

  • django/core/template/loaders/eggs.py

     
    1818        pkg_name = 'templates/' + template_name + TEMPLATE_FILE_EXTENSION
    1919        for app in INSTALLED_APPS:
    2020            try:
    21                 return resource_string(app, pkg_name)
     21                return (resource_string(app, pkg_name), 'egg:%s:%s ' % (app, pkg_name))
    2222            except:
    2323                pass
    2424    raise TemplateDoesNotExist, template_name

  • django/core/template/loader.py

     
    77#
    88# name is the template name.
    99# dirs is an optional list of directories to search instead of TEMPLATE_DIRS.
     10# The loader should return a tuple of (template_source, path). The path returned
     11# will be shown to the user for debugging purposes, so it should identify where the template
     12# was loaded from. 
    1013#
    1114# Each loader should have an "is_usable" attribute set. This is a boolean that
    1215# specifies whether the loader can be used in this Python installation. Each
    … …  
    1720# installed, because pkg_resources is necessary to read eggs.
    1821
    1922from django.core.exceptions import ImproperlyConfigured
    20 from django.core.template import Template, Context, Node, TemplateDoesNotExist, TemplateSyntaxError, resolve_variable_with_filters, register_tag
     23from django.core.template import Template, Context, Node, TemplateDoesNotExist, TemplateSyntaxError, resolve_variable_with_filters, register_tag, UNKNOWN_SOURCE
    2124from django.conf.settings import TEMPLATE_LOADERS
    2225
    2326template_source_loaders = []
    … …  
    3841    else:
    3942        template_source_loaders.append(func)
    4043
    41 def load_template_source(name, dirs=None):
     44def find_template_source(name, dirs=None):
    4245    for loader in template_source_loaders:
    4346        try:
    4447            return loader(name, dirs)
    … …  
    4649            pass
    4750    raise TemplateDoesNotExist, name
    4851
     52def load_template_source(name, dirs=None):
     53    find_template_source(name, dirs)[0]
     54
    4955class ExtendsError(Exception):
    5056    pass
    5157
    … …  
    5460    Returns a compiled Template object for the given template name,
    5561    handling template inheritance recursively.
    5662    """
    57     return get_template_from_string(load_template_source(template_name))
     63    return get_template_from_string(*find_template_source(template_name))
    5864
    59 def get_template_from_string(source):
     65def get_template_from_string(source, filename=UNKNOWN_SOURCE):
    6066    """
    6167    Returns a compiled Template object for the given template code,
    6268    handling template inheritance recursively.
    6369    """
    64     return Template(source)
     70    return Template(source, filename)
    6571
    6672def render_to_string(template_name, dictionary=None, context_instance=None):
    6773    """
    6874    Loads the given template_name and renders it with the given dictionary as
    6975    context. The template_name may be a string to load a single template using
    7076    get_template, or it may be a tuple to use select_template to find one of
    71     the templates in the list. Returns a string.
     77    the templates in the list.  Returns a string.
    7278    """
    7379    dictionary = dictionary or {}
    7480    if isinstance(template_name, (list, tuple)):
    … …  
    134140                error_msg += " Got this from the %r variable." % self.parent_name_var
    135141            raise TemplateSyntaxError, error_msg
    136142        try:
    137             return get_template_from_string(load_template_source(parent, self.template_dirs))
     143            return get_template_from_string(*find_template_source(parent, self.template_dirs))
    138144        except TemplateDoesNotExist:
    139145            raise TemplateSyntaxError, "Template %r cannot be extended, because it doesn't exist" % parent
    140146
    … …  
    186192
    187193    This tag may be used in two ways: ``{% extends "base" %}`` (with quotes)
    188194    uses the literal value "base" as the name of the parent template to extend,
    189     or ``{% entends variable %}`` uses the value of ``variable`` as the name
     195    or ``{% extends variable %}`` uses the value of ``variable`` as the name
    190196    of the parent template to extend.
    191197    """
    192198    bits = token.contents.split()

  • tests/othertests/templates.py

     
    217217    'exception04': ("{% extends 'inheritance17' %}{% block first %}{% echo 400 %}5678{% endblock %}", {}, template.TemplateSyntaxError),
    218218}
    219219
     220
     221# This replaces the standard template loader.
    220222def test_template_loader(template_name, template_dirs=None):
    221     "A custom template loader that loads the unit-test templates."
    222223    try:
    223         return TEMPLATE_TESTS[template_name][0]
     224        return ( TEMPLATE_TESTS[template_name][0] , "test:%s" % template_name )
    224225    except KeyError:
    225226        raise template.TemplateDoesNotExist, template_name
    226227
    … …  
    228229    # Register our custom template loader.
    229230    old_template_loaders = loader.template_source_loaders
    230231    loader.template_source_loaders = [test_template_loader]
    231 
     232   
    232233    failed_tests = []
    233234    tests = TEMPLATE_TESTS.items()
    234235    tests.sort()
Back to Top