| 1241 | return self.text_tokenizer.convert_token_to_id(token) |
| 1242 | |
| 1243 | def DecodeIds(self, Ids, type_token=False): |
| 1244 | if type_token: |
| 1245 | return ' '.join(Id.token if isinstance(Id, TypeToken) else self.type_id_map[Id].token for Id in Ids) |
| 1246 | if isinstance(Ids, Tokenization): |
| 1247 | Ids = Ids.tokenization |
| 1248 | Ids = list(map(int, Ids)) |
| 1249 | pieces = [] |
| 1250 | last = 0 |
| 1251 | for i, token_id in enumerate(Ids): |
| 1252 | if token_id in self.command_id_map: |
| 1253 | pieces.append(Ids[last: i]) |
| 1254 | pieces.append(token_id) |
| 1255 | last = i + 1 |
| 1256 | pieces.append(Ids[last:]) |
| 1257 | text = "" |
| 1258 | for piece in pieces: |
| 1259 | if isinstance(piece, int): |
| 1260 | text += self.command_id_map[piece].token |
| 1261 | elif piece: |
| 1262 | text += self.text_tokenizer.decode(piece) |
| 1263 | return text |
| 1264 | |
| 1265 | def DecodeTokens(self, Tokens, type_token=False): |
| 1266 | if type_token: |