(self, token, type_token=False)
| 1113 | return self.text_tokenizer.decoder[Id] |
| 1114 | |
| 1115 | def TokenToId(self, token, type_token=False): |
| 1116 | if isinstance(token, (TypeToken, CommandToken)): |
| 1117 | return token.Id |
| 1118 | if type_token: |
| 1119 | return self.type_token_map[token].Id |
| 1120 | return self.text_tokenizer.encoder[token] |
| 1121 | |
| 1122 | def DecodeIds(self, Ids, type_token=False): |
| 1123 | if type_token: |