convert Id to sentencpiece token
(self, Id, type_token=False)
| 883 | return Tokenization(tokens, processed_text, text, asIds=False) |
| 884 | |
| 885 | def IdToToken(self, Id, type_token=False): |
| 886 | """convert Id to sentencpiece token""" |
| 887 | if isinstance(Id, (TypeToken, CommandToken)): |
| 888 | return Id.token |
| 889 | if type_token: |
| 890 | return self.type_id_map[Id].token |
| 891 | if Id in self.command_id_map: |
| 892 | return self.command_id_map[Id].token |
| 893 | return self.text_tokenizer.ids_to_tokens[Id] |
| 894 | |
| 895 | def TokenToId(self, token, type_token=False): |
| 896 | """convert sentencpiece token to Id""" |
nothing calls this directly
no outgoing calls
no test coverage detected