Method encode

axlearn/common/input_grain_text.py:34–36

Tokenizes string to an int sequence.

(self, s: str)

        """BOS token ID."""

    def encode(self, s: str) -> Sequence[int]:
        """Tokenizes string to an int sequence."""
        raise NotImplementedError(type(self))

    # This API is useful for e.g. `num_bytes`.
    def _decode(self, ids: Sequence[int]) -> str: