Function lex

numpy/_build_utils/tempita/_tempita.py:560–627

Lex a string into chunks: >>> lex('hey') ['hey'] >>> lex('hey {{you}}') ['hey ', ('you', (1, 7))] >>> lex('hey {{') Traceback (most recent call last): ... TemplateError: No }} to finish last expression at line 1 column 7

(s, name=None, trim_whitespace=True, line_offset=0, delimiters=None)

def lex(s, name=None, trim_whitespace=True, line_offset=0, delimiters=None):
    """
    Lex a string into chunks:

        >>> lex('hey')
        ['hey']
        >>> lex('hey {{you}}')
        ['hey ', ('you', (1, 7))]
        >>> lex('hey {{')
        Traceback (most recent call last):
            ...
        TemplateError: No }} to finish last expression at line 1 column 7
        >>> lex('hey }}')
        Traceback (most recent call last):
            ...
        TemplateError: }} outside expression at line 1 column 7
        >>> lex('hey {{ {{')
        Traceback (most recent call last):
            ...
        TemplateError: {{ inside expression at line 1 column 10

    """
    if delimiters is None:
        delimiters = (
            Template.default_namespace["start_braces"],
            Template.default_namespace["end_braces"],
        )
    in_expr = False
    chunks = []
    last = 0
    last_pos = (line_offset + 1, 1)

    token_re = re.compile(
        rf"{re.escape(delimiters[0])}|{re.escape(delimiters[1])}"
    )
    for match in token_re.finditer(s):
        expr = match.group(0)
        pos = find_position(s, match.end(), last, last_pos)
        if expr == delimiters[0] and in_expr:
            raise TemplateError(
                f"{delimiters[0]} inside expression", position=pos, name=name
            )
        elif expr == delimiters[1] and not in_expr:
            raise TemplateError(
                f"{delimiters[1]} outside expression", position=pos, name=name
            )
        if expr == delimiters[0]:
            part = s[last:match.start()]
            if part:
                chunks.append(part)
            in_expr = True
        else:
            chunks.append((s[last: match.start()], last_pos))
            in_expr = False
        last = match.end()
        last_pos = pos
    if in_expr:
        raise TemplateError(