convert text to sentencepiece Ids
(self, text, process_fn=None)
| 745 | print('sentencepiece model written to ' + self.spm_model, flush=True) |
| 746 | |
| 747 | def EncodeAsIds(self, text, process_fn=None): |
| 748 | """convert text to sentencepiece Ids""" |
| 749 | processed_text = text |
| 750 | if process_fn is not None: |
| 751 | processed_text = process_fn(processed_text) |
| 752 | tokens = self.sp.EncodeAsIds(processed_text) |
| 753 | return Tokenization(tokens, processed_text, text) |
| 754 | |
| 755 | def EncodeAsTokens(self, text, process_fn=None): |
| 756 | """convert text to sentencepiece tokens""" |
nothing calls this directly
no test coverage detected