Function tokenize

markdown_it/rules_inline/emphasis.py:8–37

Insert each marker as a separate text token, and add it to delimiter list

(state: StateInline, silent: bool)

def tokenize(state: StateInline, silent: bool) -> bool:
    """Insert each marker as a separate text token, and add it to delimiter list"""
    start = state.pos
    marker = state.src[start]

    if silent:
        return False

    if marker not in ("_", "*"):
        return False

    scanned = state.scanDelims(state.pos, marker == "*")

    for _ in range(scanned.length):
        token = state.push("text", "", 0)
        token.content = marker
        state.delimiters.append(
            Delimiter(
                marker=ord(marker),
                length=scanned.length,
                token=len(state.tokens) - 1,
                end=-1,
                open=scanned.can_open,
                close=scanned.can_close,
            )
        )

    state.pos += scanned.length

    return True


def _postProcess(state: StateInline, delimiters: list[Delimiter]) -> None: