Method eos_token_id

bert/tokenization_utils_base.py:821–825

Id of the end of sentence token in the vocabulary. Log an error if used while not having been set.

(self)

    @property
    def eos_token_id(self):
        """ Id of the end of sentence token in the vocabulary. Log an error if used while not having been set. """
        if self._eos_token is None:
            return None
        return self.convert_tokens_to_ids(self.eos_token)

    @property
    def unk_token_id(self):