Method sep_token_id

bert/tokenization_utils_base.py:835–839

Id of the separation token in the vocabulary. E.g. separate context and query in an input sequence. Log an error if used while not having been set.

(self)

    @property
    def sep_token_id(self):
        """ Id of the separation token in the vocabulary. E.g. separate context and query in an input sequence. Log an error if used while not having been set. """
        if self._sep_token is None:
            return None
        return self.convert_tokens_to_ids(self.sep_token)

    @property
    def pad_token_id(self):