convert wordpiece token to Id
(self, text, process_fn=None)
| 875 | return ids |
| 876 | |
| 877 | def EncodeAsTokens(self, text, process_fn=None): |
| 878 | """convert wordpiece token to Id""" |
| 879 | processed_text = text |
| 880 | if process_fn is not None: |
| 881 | processed_text = process_fn(processed_text) |
| 882 | tokens = self.text_tokenizer.tokenize(processed_text) |
| 883 | return Tokenization(tokens, processed_text, text, asIds=False) |
| 884 | |
| 885 | def IdToToken(self, Id, type_token=False): |
| 886 | """convert Id to sentencpiece token""" |
nothing calls this directly
no test coverage detected