Method encode

open_mythos/tokenizer.py:42–52

Encode input text into a list of token IDs. Args: text (str): The input text string to tokenize. Returns: list[int]: List of integer token IDs representing the input text.

(self, text: str)

        return self.tokenizer.vocab_size

    def encode(self, text: str) -> list[int]:
        """
        Encode input text into a list of token IDs.

        Args:
            text (str): The input text string to tokenize.

        Returns:
            list[int]: List of integer token IDs representing the input text.
        """
        return self.tokenizer.encode(text, add_special_tokens=False)

    def decode(self, token_ids: list[int]) -> str:
        """