(self, text, process_fn=None)
| 1212 | return ids |
| 1213 | |
| 1214 | def EncodeAsTokens(self, text, process_fn=None): |
| 1215 | processed_text = text |
| 1216 | if process_fn is not None: |
| 1217 | processed_text = process_fn(processed_text) |
| 1218 | tokens = self.text_tokenizer.tokenize(processed_text) |
| 1219 | tokenization = Tokenization(tokens, processed_text, text, asIds=False) |
| 1220 | tokenization.set_command_tokens(self._command_tokens) |
| 1221 | return tokenization |
| 1222 | # return Tokenization(tokens, processed_text, text, asIds=False) |
| 1223 | |
| 1224 | def IdToToken(self, Id, type_token=False): |
| 1225 | if isinstance(Id, (TypeToken, CommandToken)): |
nothing calls this directly
no test coverage detected