Method encode

lmdeploy/tokenizer.py:465–483

Tokenize a prompt. Args: s: a prompt. add_bos: Whether to add ``bos`` token id when encoding the prompt. add_special_tokens: Whether or not to add special tokens when encoding the prompt. Returns: list[int]: token ids.

(self, s: str, add_bos: bool = True, add_special_tokens: bool = True, **kwargs)

        return self.model.get_vocab()

    def encode(self, s: str, add_bos: bool = True, add_special_tokens: bool = True, **kwargs):
        """Tokenize a prompt.

        Args:
            s: a prompt.
            add_bos: Whether to add ``bos`` token id when encoding the prompt.
            add_special_tokens: Whether or not to add special tokens
                when encoding the prompt.

        Returns:
            list[int]: token ids.
        """
        encoded = self.model.encode(s, add_bos, add_special_tokens, **kwargs)
        if encoded[:2] == [self.bos_token_id] * 2:
            self.logger.warning(f'Detected duplicate bos token {self.bos_token_id} in prompt, '
                                'this will likely reduce response quality, one of them will be'
                                'removed')
            encoded = encoded[1:]
        return encoded

    def decode(
        self,