converts sentencepiece tokens to a text string
(self, Tokens)
| 775 | return self.sp.DecodeIds(Ids) |
| 776 | |
| 777 | def DecodeTokens(self, Tokens): |
| 778 | """converts sentencepiece tokens to a text string""" |
| 779 | if isinstance(Tokens, Tokenization): |
| 780 | Tokens = Tokens.tokenization |
| 781 | return self.sp.DecodeTokens(Tokens) |
| 782 | |
| 783 | |
| 784 | class BertWordPieceTokenizer(Tokenizer): |
nothing calls this directly
no test coverage detected