Ticket #603: django-template-errors-3.patch

File django-template-errors-3.patch, 16.1 KB (added by rjwittams, 19 years ago)

moved files with template reorg, fixes lexing bug.

  • django/core/template/__init__.py

     
    5555'\n<html>\n\n</html>\n'
    5656"""
    5757import re
    58 from django.conf.settings import DEFAULT_CHARSET
     58from django.conf.settings import DEFAULT_CHARSET, DEBUG
    5959
    6060__all__ = ('Template','Context','compile_string')
    6161
    … …  
    7474
    7575ALLOWED_VARIABLE_CHARS = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_.'
    7676
     77#What to report as the origin of templates that come from non file sources (eg strings)
     78UNKNOWN_SOURCE="<unknown source>"
     79
     80#match starts of lines
     81newline_re = re.compile("^", re.M);
     82
    7783# match a variable or block tag and capture the entire tag, including start/end delimiters
    7884tag_re = re.compile('(%s.*?%s|%s.*?%s)' % (re.escape(BLOCK_TAG_START), re.escape(BLOCK_TAG_END),
    7985                                          re.escape(VARIABLE_TAG_START), re.escape(VARIABLE_TAG_END)))
    … …  
    102108    pass
    103109
    104110class Template:
    105     def __init__(self, template_string):
     111    def __init__(self, template_string, filename=UNKNOWN_SOURCE):
    106112        "Compilation stage"
    107         self.nodelist = compile_string(template_string)
    108 
     113        self.nodelist = compile_string(template_string, filename)
     114       
    109115    def __iter__(self):
    110116        for node in self.nodelist:
    111117            for subnode in node:
    … …  
    115121        "Display stage -- can be called many times"
    116122        return self.nodelist.render(context)
    117123
    118 def compile_string(template_string):
     124def compile_string(template_string, filename):
    119125    "Compiles template_string into NodeList ready for rendering"
    120     tokens = tokenize(template_string)
    121     parser = Parser(tokens)
     126    if DEBUG:
     127        lexer_factory = DebugLexer
     128        parser_factory = DebugParser
     129    else:
     130        lexer_factory = Lexer
     131        parser_factory = Parser
     132       
     133    lexer = lexer_factory(template_string, filename)
     134    parser = parser_factory(lexer.tokenize())
    122135    return parser.parse()
    123136
    124137class Context:
    … …  
    177190            {TOKEN_TEXT:'Text', TOKEN_VAR:'Var', TOKEN_BLOCK:'Block'}[self.token_type],
    178191            self.contents[:20].replace('\n', '')
    179192            )
     193           
     194    def __repr__(self):
     195        return '<%s token: "%s">' % (
     196            {TOKEN_TEXT:'Text', TOKEN_VAR:'Var', TOKEN_BLOCK:'Block'}[self.token_type],
     197            self.contents[:].replace('\n', '')
     198            )
    180199
    181 def tokenize(template_string):
    182     "Return a list of tokens from a given template_string"
    183     # remove all empty strings, because the regex has a tendency to add them
    184     bits = filter(None, tag_re.split(template_string))
    185     return map(create_token, bits)
     200class Lexer(object):
     201    def __init__(self, template_string, filename):
     202        self.template_string = template_string
     203        self.filename = filename
     204   
     205    def tokenize(self):
     206        "Return a list of tokens from a given template_string"
     207        bits = filter(None, tag_re.split(self.template_string))
     208        return map(self.create_token, bits)
     209       
     210    def create_token(self,token_string):
     211        "Convert the given token string into a new Token object and return it"
     212        if token_string.startswith(VARIABLE_TAG_START):
     213            token = Token(TOKEN_VAR, token_string[len(VARIABLE_TAG_START):-len(VARIABLE_TAG_END)].strip())
     214        elif token_string.startswith(BLOCK_TAG_START):
     215            token = Token(TOKEN_BLOCK, token_string[len(BLOCK_TAG_START):-len(BLOCK_TAG_END)].strip())
     216        else:
     217            token = Token(TOKEN_TEXT, token_string)
     218        return token
    186219
    187 def create_token(token_string):
    188     "Convert the given token string into a new Token object and return it"
    189     if token_string.startswith(VARIABLE_TAG_START):
    190         return Token(TOKEN_VAR, token_string[len(VARIABLE_TAG_START):-len(VARIABLE_TAG_END)].strip())
    191     elif token_string.startswith(BLOCK_TAG_START):
    192         return Token(TOKEN_BLOCK, token_string[len(BLOCK_TAG_START):-len(BLOCK_TAG_END)].strip())
    193     else:
    194         return Token(TOKEN_TEXT, token_string)
     220class DebugLexer(Lexer):
     221    def __init__(self, template_string, filename):
     222        super(DebugLexer,self).__init__(template_string, filename)
    195223
    196 class Parser:
     224    def find_linebreaks(self, template_string):
     225        for match in newline_re.finditer(template_string):
     226            yield match.start()
     227
     228    def tokenize(self):
     229        "Return a list of tokens from a given template_string"
     230       
     231        token_tups = []
     232        upto = 0
     233        line = 1
     234        #TODO:Py2.4 generator expression
     235        linebreaks = self.find_linebreaks(self.template_string)
     236        next_linebreak = linebreaks.next()
     237       
     238
     239        for match in tag_re.finditer(self.template_string):
     240            start, end = match.span()
     241            if start > upto:       
     242                token_tups.append( (self.template_string[upto:start], line) )
     243                upto = start
     244               
     245                while next_linebreak <= upto:
     246                    try:
     247                        next_linebreak = linebreaks.next()
     248                        line += 1
     249                    except StopIteration:
     250                        break
     251           
     252            token_tups.append( (self.template_string[start:end], line) )
     253            upto = end
     254   
     255            while next_linebreak <= upto:
     256                try:
     257                    next_linebreak = linebreaks.next()
     258                    line += 1
     259                except StopIteration:
     260                    break
     261
     262        last_bit = self.template_string[upto:]
     263        if len(last_bit):
     264           token_tups.append( (last_bit, line) )
     265       
     266        return [ self.create_token(tok, (self.filename, line)) for tok, line in token_tups]
     267
     268
     269    def create_token(self, token_string, source):
     270        token = super(DebugLexer, self).create_token(token_string)
     271        token.source = source
     272        return token
     273
     274class Parser(object):
    197275    def __init__(self, tokens):
    198276        self.tokens = tokens
     277       
    199278
    200279    def parse(self, parse_until=[]):
    201280        nodelist = NodeList()
    202281        while self.tokens:
    203282            token = self.next_token()
    204283            if token.token_type == TOKEN_TEXT:
    205                 nodelist.append(TextNode(token.contents))
     284                self.extend_nodelist(nodelist, TextNode(token.contents), token)
    206285            elif token.token_type == TOKEN_VAR:
    207286                if not token.contents:
    208                     raise TemplateSyntaxError, "Empty variable tag"
    209                 nodelist.append(VariableNode(token.contents))
     287                    self.empty_variable(token)
     288                self.extend_nodelist(nodelist, VariableNode(token.contents), token)
    210289            elif token.token_type == TOKEN_BLOCK:
    211290                if token.contents in parse_until:
    212291                    # put token back on token list so calling code knows why it terminated
    … …  
    215294                try:
    216295                    command = token.contents.split()[0]
    217296                except IndexError:
    218                     raise TemplateSyntaxError, "Empty block tag"
     297                    self.empty_block_tag(token)
     298               
     299                # execute callback function for this tag and append resulting node
     300                self.enter_command(command, token);
    219301                try:
    220                     # execute callback function for this tag and append resulting node
    221                     nodelist.append(registered_tags[command](self, token))
     302                    compile_func = registered_tags[command]
    222303                except KeyError:
    223                     raise TemplateSyntaxError, "Invalid block tag: '%s'" % command
     304                    self.invalid_block_tag(token, command)
     305                   
     306                self.extend_nodelist(nodelist, compile_func(self, token), token)
     307                self.exit_command();
     308               
    224309        if parse_until:
    225             raise TemplateSyntaxError, "Unclosed tag(s): '%s'" % ', '.join(parse_until)
     310            self.unclosed_block_tag(token, parse_until)
     311           
    226312        return nodelist
    227313
     314    def extend_nodelist(self, nodelist, node, token):
     315        nodelist.append(node)
     316
     317    def enter_command(self, command, token):
     318        pass
     319       
     320    def exit_command(self):
     321        pass
     322
     323    def empty_variable(self, token):
     324        raise TemplateSyntaxError, "Empty variable tag"
     325   
     326    def empty_block_tag(self, token):
     327        raise TemplateSyntaxError, "Empty block tag"
     328   
     329    def invalid_block_tag(self, token, command):
     330        raise TemplateSyntaxError, "Invalid block tag: %s" % (command)
     331   
     332    def unclosed_block_tag(self, token, parse_until):
     333        raise TemplateSyntaxError, "Unclosed tags: %s " %  ', '.join(parse_until)
     334       
    228335    def next_token(self):
    229336        return self.tokens.pop(0)
    230337
    … …  
    234341    def delete_first_token(self):
    235342        del self.tokens[0]
    236343
     344
     345class DebugParser(Parser):
     346    def __init__(self, lexer):
     347        super(DebugParser, self).__init__(lexer)
     348        self.command_stack = []
     349
     350    def enter_command(self, command, token):
     351        self.command_stack.append( (command, token.source) )
     352       
     353    def exit_command(self):
     354        self.command_stack.pop()
     355
     356    def format_source(self, source):
     357        return "at %s, line %d" % source
     358
     359    def extend_nodelist(self, nodelist, node, token):
     360        node.source = token.source
     361        super(DebugParser, self).extend_nodelist(nodelist, node, token)
     362
     363    def empty_variable(self, token):
     364        raise TemplateSyntaxError, "Empty variable tag %s" % self.format_source(token.source)
     365   
     366    def empty_block_tag(self, token):
     367        raise TemplateSyntaxError, "Empty block tag %s" % self.format_source(token.source)
     368   
     369    def invalid_block_tag(self, token, command):
     370        raise TemplateSyntaxError, "Invalid block tag: '%s' %s" % (command, self.format_source(token.source))
     371   
     372    def unclosed_block_tag(self, token, parse_until):
     373        (command, (file,line)) = self.command_stack.pop()
     374        msg = "Unclosed tag '%s' starting at %s, line %d. Looking for one of: %s " % \
     375              (command, file, line, ', '.join(parse_until) )
     376        raise TemplateSyntaxError, msg
     377
    237378class FilterParser:
    238379    """Parse a variable token and its optional filters (all as a single string),
    239380       and return a list of tuples of the filter name and arguments.

  • django/core/template/loaders/filesystem.py

     
    1111    for template_dir in template_dirs:
    1212        filepath = os.path.join(template_dir, template_name) + TEMPLATE_FILE_EXTENSION
    1313        try:
    14             return open(filepath).read()
     14            return (open(filepath).read(), filepath)
    1515        except IOError:
    1616            tried.append(filepath)
    1717    if template_dirs:

  • django/core/template/loaders/eggs.py

     
    1818        pkg_name = 'templates/' + name + TEMPLATE_FILE_EXTENSION
    1919        for app in INSTALLED_APPS:
    2020            try:
    21                 return resource_string(app, pkg_name)
     21                return (resource_string(app, pkg_name), 'egg:%s:%s ' % (app, pkg_name))
    2222            except:
    2323                pass
    2424    raise TemplateDoesNotExist, name

  • django/core/template/loader.py

     
    77#
    88# name is the template name.
    99# dirs is an optional list of directories to search instead of TEMPLATE_DIRS.
     10# The loader should return a tuple of (template_source, path). The path returned
     11# will be shown to the user for debugging purposes, so it should identify where the template
     12# was loaded from. 
    1013#
    1114# Each loader should have an "is_usable" attribute set. This is a boolean that
    1215# specifies whether the loader can be used in this Python installation. Each
    … …  
    1720# installed, because pkg_resources is necessary to read eggs.
    1821
    1922from django.core.exceptions import ImproperlyConfigured
    20 from django.core.template import Template, Context, Node, TemplateDoesNotExist, TemplateSyntaxError, resolve_variable_with_filters, register_tag
     23from django.core.template import Template, Context, Node, TemplateDoesNotExist, TemplateSyntaxError, resolve_variable_with_filters, register_tag, UNKNOWN_SOURCE
    2124from django.conf.settings import TEMPLATE_LOADERS
    2225
    2326template_source_loaders = []
    … …  
    3841    else:
    3942        template_source_loaders.append(func)
    4043
    41 def load_template_source(name, dirs=None):
     44def find_template_source(name, dirs=None):
    4245    for loader in template_source_loaders:
    4346        try:
    4447            return loader(name, dirs)
    … …  
    4649            pass
    4750    raise TemplateDoesNotExist, name
    4851
     52def load_template_source(name, dirs=None):
     53    find_template_source(name, dirs)[0]
     54
    4955class ExtendsError(Exception):
    5056    pass
    5157
    … …  
    5460    Returns a compiled Template object for the given template name,
    5561    handling template inheritance recursively.
    5662    """
    57     return get_template_from_string(load_template_source(template_name))
     63    return get_template_from_string(*find_template_source(template_name))
    5864
    59 def get_template_from_string(source):
     65def get_template_from_string(source, filename=UNKNOWN_SOURCE):
    6066    """
    6167    Returns a compiled Template object for the given template code,
    6268    handling template inheritance recursively.
    6369    """
    64     return Template(source)
     70    return Template(source, filename)
    6571
    6672def render_to_string(template_name, dictionary=None, context_instance=None):
    6773    """
    6874    Loads the given template_name and renders it with the given dictionary as
    6975    context. The template_name may be a string to load a single template using
    7076    get_template, or it may be a tuple to use select_template to find one of
    71     the templates in the list. Returns a string.
     77    the templates in the list.  Returns a string.
    7278    """
    7379    dictionary = dictionary or {}
    7480    if isinstance(template_name, (list, tuple)):
    … …  
    134140                error_msg += " Got this from the %r variable." % self.parent_name_var
    135141            raise TemplateSyntaxError, error_msg
    136142        try:
    137             return get_template_from_string(load_template_source(parent, self.template_dirs))
     143            return get_template_from_string(*find_template_source(parent, self.template_dirs))
    138144        except TemplateDoesNotExist:
    139145            raise TemplateSyntaxError, "Template %r cannot be extended, because it doesn't exist" % parent
    140146
    … …  
    186192
    187193    This tag may be used in two ways: ``{% extends "base" %}`` (with quotes)
    188194    uses the literal value "base" as the name of the parent template to extend,
    189     or ``{% entends variable %}`` uses the value of ``variable`` as the name
     195    or ``{% extends variable %}`` uses the value of ``variable`` as the name
    190196    of the parent template to extend.
    191197    """
    192198    bits = token.contents.split()

  • tests/othertests/templates.py

     
    217217    'exception04': ("{% extends 'inheritance17' %}{% block first %}{% echo 400 %}5678{% endblock %}", {}, template.TemplateSyntaxError),
    218218}
    219219
     220
     221# This replaces the standard template loader.
    220222def test_template_loader(template_name, template_dirs=None):
    221     "A custom template loader that loads the unit-test templates."
    222223    try:
    223         return TEMPLATE_TESTS[template_name][0]
     224        return ( TEMPLATE_TESTS[template_name][0] , "test:%s" % template_name )
    224225    except KeyError:
    225226        raise template.TemplateDoesNotExist, template_name
    226227
    … …  
    228229    # Register our custom template loader.
    229230    old_template_loaders = loader.template_source_loaders
    230231    loader.template_source_loaders = [test_template_loader]
    231 
     232   
    232233    failed_tests = []
    233234    tests = TEMPLATE_TESTS.items()
    234235    tests.sort()
Back to Top