(self, text, is_split_into_words=False, **kwargs)
| 302 | return vocab_file, merge_file |
| 303 | |
| 304 | def prepare_for_tokenization(self, text, is_split_into_words=False, **kwargs): |
| 305 | add_prefix_space = kwargs.pop("add_prefix_space", self.add_prefix_space) |
| 306 | if is_split_into_words or add_prefix_space: |
| 307 | text = " " + text |
| 308 | return (text, kwargs) |
| 309 | |
| 310 | def decode( |
| 311 | self, |
nothing calls this directly
no outgoing calls
no test coverage detected