Method _convert_token_to_id

models/tokenization_moss.py:261–263

Converts a token (str) in an id using the vocab.

(self, token)

        return bpe_tokens

    def _convert_token_to_id(self, token):
        """Converts a token (str) in an id using the vocab."""
        return self.encoder.get(token, self.encoder.get(self.unk_token))

    def _convert_id_to_token(self, index):
        """Converts an index (integer) in a token (str) using the vocab."""