convert token to Id accounting for command and type tokens
(self, token, type_token=False)
| 415 | return self.text_tokenizer.IdToToken(Id - self.num_command_tokens) |
| 416 | |
| 417 | def TokenToId(self, token, type_token=False): |
| 418 | """convert token to Id accounting for command and type tokens""" |
| 419 | if isinstance(token, (TypeToken, CommandToken)): |
| 420 | return token.Id |
| 421 | if type_token: |
| 422 | return self.type_token_map[token].Id |
| 423 | if token in self.command_token_map: |
| 424 | return self.command_token_map[token].Id |
| 425 | return self.text_tokenizer.TokenToId(token) + self.num_command_tokens |
| 426 | |
| 427 | def DecodeIds(self, Ids, type_token=False): |
| 428 | """ |