Method encode

tests/conftest.py:33–44

Encode text to token ids (simple simulation).

(self, text: str, add_special_tokens: bool = True)

        self.bos_token_id = 1

    def encode(self, text: str, add_special_tokens: bool = True) -> List[int]:
        """Encode text to token ids (simple simulation)."""
        # Simple simulation: each word becomes a token
        tokens = []
        if add_special_tokens:
            tokens.append(self.bos_token_id)
        # Simulate tokenization by splitting on spaces
        for i, word in enumerate(text.split()):
            # Use hash to get a consistent token id for each word
            token_id = (hash(word) % (self.vocab_size - 10)) + 10
            tokens.append(token_id)
        return tokens

    def decode(
        self,