Returns a tokenized string.
(self, text)
| 80 | return vocab |
| 81 | |
| 82 | def _tokenize(self, text): |
| 83 | """Returns a tokenized string.""" |
| 84 | return self.sp_model.encode(text, out_type=str) |
| 85 | |
| 86 | def _convert_token_to_id(self, token): |
| 87 | """Converts a token (str) in an id using the vocab.""" |
nothing calls this directly
no outgoing calls
no test coverage detected