Method _convert_token_to_id

bert/tokenization_bert.py:219–221

Converts a token (str) in an id using the vocab.

(self, token)

        return split_tokens

    def _convert_token_to_id(self, token):
        """ Converts a token (str) in an id using the vocab. """
        return self.vocab.get(token, self.vocab.get(self.unk_token))

    def _convert_id_to_token(self, index):
        """Converts an index (integer) in a token (str) using the vocab."""