Method decode

ms_agent/utils/tokenizer_util.py:32–43

Decode a list of token IDs back into a natural text string. Args: token_ids: List of token IDs to decode. Returns: Decoded text string.

(self, token_ids: List[int])

        return self.tokenizer.encode(content.strip())

    def decode(self, token_ids: List[int]) -> str:
        """Decode a list of token IDs back into a natural text string.

        Args:
            token_ids: List of token IDs to decode.

        Returns:
            Decoded text string.
        """
        if not token_ids:
            return ''
        return self.tokenizer.decode(token_ids, skip_special_tokens=True)

    def segment(self, content: str) -> List[str]:
        """Tokenize text into a list of token strings suitable for BM25-like algorithms indexing/retrieval.