convert text to sentencepiece tokens
(self, text, process_fn=None)
| 753 | return Tokenization(tokens, processed_text, text) |
| 754 | |
| 755 | def EncodeAsTokens(self, text, process_fn=None): |
| 756 | """convert text to sentencepiece tokens""" |
| 757 | processed_text = text |
| 758 | if process_fn is not None: |
| 759 | processed_text = process_fn(processed_text) |
| 760 | tokens = self.sp.EncodeAsTokens(processed_text) |
| 761 | return Tokenization(tokens, processed_text, text, asIds=False) |
| 762 | |
| 763 | def IdToToken(self, Id): |
| 764 | """convert Id to sentencpiece token""" |
nothing calls this directly
no test coverage detected