Class Token

sphinx/pycode/parser.py:105–141

Better token wrapper for tokenize module.

class Token:
    """Better token wrapper for tokenize module."""

    def __init__(
        self,
        kind: int,
        value: Any,
        start: tuple[int, int],
        end: tuple[int, int],
        source: str,
    ) -> None:
        self.kind = kind
        self.value = value
        self.start = start
        self.end = end
        self.source = source

    def __eq__(self, other: object) -> bool:
        if isinstance(other, int):
            return self.kind == other
        elif isinstance(other, str):
            return self.value == other
        elif isinstance(other, (list, tuple)):
            return [self.kind, self.value] == list(other)
        elif other is None:
            return False
        else:
            raise ValueError('Unknown value: %r' % other)

    def __hash__(self) -> int:
        return hash((self.kind, self.value, self.start, self.end, self.source))

    def match(self, *conditions: Any) -> bool:
        return any(self == candidate for candidate in conditions)

    def __repr__(self) -> str:
        return f'<Token kind={tokenize.tok_name[self.kind]!r} value={self.value.strip()!r}>'


class TokenProcessor: