Method build_inputs_with_special_tokens

models/tokenization_moss.py:238–249

(self, token_ids_0, token_ids_1=None)

        return word

    def build_inputs_with_special_tokens(self, token_ids_0, token_ids_1=None):
        if self.add_bos_token:
            bos_token_ids = [self.bos_token_id]
        else:
            bos_token_ids = []

        output = bos_token_ids + token_ids_0

        if token_ids_1 is None:
            return output

        return output + bos_token_ids + token_ids_1

    def _tokenize(self, text):
        """Tokenize a string."""