(self, Ids, type_token=False)
| 1120 | return self.text_tokenizer.encoder[token] |
| 1121 | |
| 1122 | def DecodeIds(self, Ids, type_token=False): |
| 1123 | if type_token: |
| 1124 | return ' '.join(Id.token if isinstance(Id, TypeToken) else self.type_id_map[Id].token for Id in Ids) |
| 1125 | if isinstance(Ids, Tokenization): |
| 1126 | Ids = Ids.tokenization |
| 1127 | return self.text_tokenizer.decode(Ids) |
| 1128 | |
| 1129 | def DecodeTokens(self, Tokens, type_token=False): |
| 1130 | if type_token: |