Convert a list or tokenization object of tokens to a text string
(self, Tokens)
| 539 | raise NotImplementedError('TextTokenizer DecodeIds not implemented') |
| 540 | |
| 541 | def DecodeTokens(self, Tokens): |
| 542 | """Convert a list or tokenization object of tokens to a text string""" |
| 543 | raise NotImplementedError('TextTokenizer DecodeTokens not implemented') |
| 544 | |
| 545 | |
| 546 | class CharacterLevelTokenizer(TextTokenizer): |
nothing calls this directly
no outgoing calls
no test coverage detected