Converts a sequence of tokens (string) in a single string.
(self, tokens)
| 225 | return self.ids_to_tokens.get(index, self.unk_token) |
| 226 | |
| 227 | def convert_tokens_to_string(self, tokens): |
| 228 | """ Converts a sequence of tokens (string) in a single string. """ |
| 229 | out_string = " ".join(tokens).replace(" ##", "").strip() |
| 230 | return out_string |
| 231 | |
| 232 | def build_inputs_with_special_tokens( |
| 233 | self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None |
nothing calls this directly
no outgoing calls
no test coverage detected