encode text using text tokenizer and shift Id values for command tokens
(self, text, process_fn=None)
| 299 | return self._text_token_vocab |
| 300 | |
| 301 | def EncodeAsIds(self, text, process_fn=None): |
| 302 | """ |
| 303 | encode text using text tokenizer and shift Id values for command tokens |
| 304 | """ |
| 305 | tokenization = self.text_tokenizer.EncodeAsIds(text, process_fn=process_fn) |
| 306 | tokenization.tokenization = [t+self.num_command_tokens for t in tokenization.tokenization] |
| 307 | tokenization.set_command_tokens(self._command_tokens) |
| 308 | return tokenization |
| 309 | |
| 310 | def EncodeAsTokens(self, text, process_fn=None): |
| 311 | """ |
no test coverage detected