Method tokenize

tools/inspector_protocol/jinja2/lexer.py:552–556

Calls tokeniter + tokenize and wraps it in a token stream.

(self, source, name=None, filename=None, state=None)

        return newline_re.sub(self.newline_sequence, value)

    def tokenize(self, source, name=None, filename=None, state=None):
        """Calls tokeniter + tokenize and wraps it in a token stream.
        """
        stream = self.tokeniter(source, name, filename, state)
        return TokenStream(self.wrap(stream, name, filename), name, filename)

    def wrap(self, stream, name=None, filename=None):
        """This is called with the stream as returned by `tokenize` and wraps