(self, text, **kwargs)
| 100 | return text |
| 101 | |
| 102 | def _tokenize(self, text, **kwargs): |
| 103 | tokens = [] |
| 104 | if self.encode_special_tokens: |
| 105 | ids = self.tokenizer.encode(text, allowed_special="all") |
| 106 | else: |
| 107 | ids = self.tokenizer.encode(text, disallowed_special=()) |
| 108 | for t in ids: |
| 109 | tokens.append(self.decoder[t]) |
| 110 | return tokens |
| 111 | |
| 112 | def _convert_token_to_id(self, token): |
| 113 | """ Converts a token (str) in an id using the vocab. """ |
nothing calls this directly
no outgoing calls
no test coverage detected