convert sentencpiece token to Id
(self, token, type_token=False)
| 893 | return self.text_tokenizer.ids_to_tokens[Id] |
| 894 | |
| 895 | def TokenToId(self, token, type_token=False): |
| 896 | """convert sentencpiece token to Id""" |
| 897 | if isinstance(token, (TypeToken, CommandToken)): |
| 898 | return token.Id |
| 899 | if type_token: |
| 900 | return self.type_token_map[token].Id |
| 901 | token = token.strip() |
| 902 | return self.text_tokenizer.vocab[token] |
| 903 | |
| 904 | def DecodeIds(self, Ids, type_token=False): |
| 905 | """converts ids to wordpiece tokens and joins them as a text string""" |
nothing calls this directly
no outgoing calls
no test coverage detected