convert text to ascii 256 characters
(self, text, process_fn=None)
| 582 | return Tokenization(tokens, processed_text, text) |
| 583 | |
| 584 | def EncodeAsTokens(self, text, process_fn=None): |
| 585 | """convert text to ascii 256 characters""" |
| 586 | processed_text = text |
| 587 | if process_fn is not None: |
| 588 | processed_text = process_fn(processed_text) |
| 589 | processed_text = str(processed_text) |
| 590 | tokens = [c for c in processed_text] |
| 591 | return Tokenization(tokens, processed_text, text, asIds=False) |
| 592 | |
| 593 | def IdToToken(self, Id): |
| 594 | """ascii index to character""" |
nothing calls this directly
no test coverage detected