Method tokenize

bpython/repl.py:1094–1190

Tokenizes a line of code, returning pygments tokens with side effects/impurities: - reads self.cpos to see what parens should be highlighted - reads self.buffer to see what came before the passed in line - sets self.highlighted_paren to (buffer_lineno, tokens_for_that

(self, s, newline=False)

        """See the flush() method docstring."""

    def tokenize(self, s, newline=False) -> list[tuple[_TokenType, str]]:
        """Tokenizes a line of code, returning pygments tokens
        with side effects/impurities:
        - reads self.cpos to see what parens should be highlighted
        - reads self.buffer to see what came before the passed in line
        - sets self.highlighted_paren to (buffer_lineno, tokens_for_that_line)
          for buffer line that should replace that line to unhighlight it,
          or None if no paren is currently highlighted
        - calls reprint_line with a buffer's line's tokens and the buffer
          lineno that has changed if line other than the current line changes
        """
        highlighted_paren = None

        source = "\n".join(self.buffer + [s])
        cursor = len(source) - self.cpos
        if self.cpos:
            cursor += 1
        stack: list[Any] = list()
        all_tokens = list(Python3Lexer().get_tokens(source))
        # Unfortunately, Pygments adds a trailing newline and strings with
        # no size, so strip them
        while not all_tokens[-1][1]:
            all_tokens.pop()
        all_tokens[-1] = (all_tokens[-1][0], all_tokens[-1][1].rstrip("\n"))
        line = pos = 0
        parens = dict(zip("{([", "})]"))
        line_tokens: list[tuple[_TokenType, str]] = list()
        saved_tokens: list[tuple[_TokenType, str]] = list()
        search_for_paren = True
        for token, value in split_lines(all_tokens):
            pos += len(value)
            if token is Token.Text and value == "\n":
                line += 1
                # Remove trailing newline
                line_tokens = list()
                saved_tokens = list()
                continue
            line_tokens.append((token, value))
            saved_tokens.append((token, value))
            if not search_for_paren:
                continue
            under_cursor = pos == cursor
            if token is Token.Punctuation:
                if value in parens:
                    if under_cursor:
                        line_tokens[-1] = (Parenthesis.UnderCursor, value)
                        # Push marker on the stack
                        stack.append((Parenthesis, value))
                    else:
                        stack.append(
                            (line, len(line_tokens) - 1, line_tokens, value)
                        )
                elif value in parens.values():
                    saved_stack = list(stack)
                    try:
                        while True:
                            opening = stack.pop()
                            if parens[opening[-1]] == value: