encode text as tokens using text tokenizer
(self, text, process_fn=None)
| 397 | raise NotImplementedError |
| 398 | |
| 399 | def EncodeAsTokens(self, text, process_fn=None): |
| 400 | """ |
| 401 | encode text as tokens using text tokenizer |
| 402 | """ |
| 403 | tokenization = self.text_tokenizer.EncodeAsTokens(text, process_fn=process_fn) |
| 404 | tokenization.set_command_tokens(self._command_tokens) |
| 405 | return tokenization |
| 406 | |
| 407 | def IdToToken(self, Id, type_token=False): |
| 408 | """convert Id to token accounting for command and type tokens""" |
no test coverage detected