Preprocess text and encode as tokens. Return a tokenization object with original text, processed text, and token tokenization.
(self, text, process_fn=None)
| 520 | raise NotImplementedError('TextTokenizer EncodeAsIds not implemented') |
| 521 | |
| 522 | def EncodeAsTokens(self, text, process_fn=None): |
| 523 | """ |
| 524 | Preprocess text and encode as tokens. Return a tokenization object with |
| 525 | original text, processed text, and token tokenization. |
| 526 | """ |
| 527 | raise NotImplementedError('TextTokenizer EncodeAsTokens not implemented') |
| 528 | |
| 529 | def IdToToken(self, Id): |
| 530 | """Convert an Id to Token. Reverse lookup of self.vocab""" |
nothing calls this directly
no outgoing calls
no test coverage detected