convert text to ascii 256 Ids
(self, text, process_fn=None)
| 573 | return self._vocab |
| 574 | |
| 575 | def EncodeAsIds(self, text, process_fn=None): |
| 576 | """convert text to ascii 256 Ids""" |
| 577 | processed_text = text |
| 578 | if process_fn is not None: |
| 579 | processed_text = process_fn(processed_text) |
| 580 | processed_text = str(processed_text) |
| 581 | tokens = [self.TokenToId(c) for c in processed_text] |
| 582 | return Tokenization(tokens, processed_text, text) |
| 583 | |
| 584 | def EncodeAsTokens(self, text, process_fn=None): |
| 585 | """convert text to ascii 256 characters""" |
nothing calls this directly
no test coverage detected