(self, token, type_token=False)
| 1234 | return self.text_tokenizer.convert_id_to_token(int(Id)) |
| 1235 | |
| 1236 | def TokenToId(self, token, type_token=False): |
| 1237 | if isinstance(token, (TypeToken, CommandToken)): |
| 1238 | return token.Id |
| 1239 | if type_token: |
| 1240 | return self.type_token_map[token].Id |
| 1241 | return self.text_tokenizer.convert_token_to_id(token) |
| 1242 | |
| 1243 | def DecodeIds(self, Ids, type_token=False): |
| 1244 | if type_token: |
no test coverage detected