Method get_token_len

opencompass/models/base.py:139–147

Get lengths of the tokenized strings. Args: prompt (str): Input string. Returns: int: Length of the input tokens

(self, prompt: str)

    @abstractmethod
    def get_token_len(self, prompt: str) -> int:
        """Get lengths of the tokenized strings.

        Args:
            prompt (str): Input string.

        Returns:
            int: Length of the input tokens
        """

    def parse_template(self, prompt_template: PromptType, mode: str) -> str:
        """Parse a prompt template, and wrap it with meta template if