Function tokenize

mitmproxy/contentviews/_view_xml_html.py:104–130

(data: str)

def tokenize(data: str) -> Iterable[Token]:
    token: Token = Text("")

    i = 0

    def readuntil(char, start, include=1):
        nonlocal i
        end = data.find(char, start)
        if end == -1:
            end = len(data)
        ret = data[i : end + include]
        i = end + include
        return ret

    while i < len(data):
        if isinstance(token, Text):
            token.data = readuntil("<", i, 0)
            if token.text:
                yield token
            token = Tag("")
        elif isinstance(token, Tag):
            token.data += readuntil(">", i, 1)
            if token.done:
                yield token
                token = Text("")
    if token.data.strip():
        yield token


def indent_text(data: str, prefix: str) -> str: