(self, Tokens, type_token=False)
| 1127 | return self.text_tokenizer.decode(Ids) |
| 1128 | |
| 1129 | def DecodeTokens(self, Tokens, type_token=False): |
| 1130 | if type_token: |
| 1131 | return ' '.join(t.token if isinstance(t, TypeToken) else t for t in Tokens) |
| 1132 | if isinstance(Tokens, Tokenization): |
| 1133 | Tokens = Tokens.tokenization |
| 1134 | return self.text_tokenizer.decode([self.TokenToId(tok) for tok in Tokens]) |
| 1135 | |
| 1136 | |
| 1137 | class ChineseSPTokenizer(Tokenizer): |