Method encode

ms_agent/utils/tokenizer_util.py:19–30

Encode text into token IDs. Args: content: Input text string. Returns: List of token IDs.

(self, content: str)

        self.tokenizer = AutoTokenizer.from_pretrained(model_id)

    def encode(self, content: str) -> List[int]:
        """Encode text into token IDs.

        Args:
            content: Input text string.

        Returns:
            List of token IDs.
        """
        if not content.strip():
            return []
        return self.tokenizer.encode(content.strip())

    def decode(self, token_ids: List[int]) -> str:
        """Decode a list of token IDs back into a natural text string.