(self, token_ids)
| 236 | return input_ids |
| 237 | |
| 238 | def detokenize(self, token_ids): |
| 239 | text = self.tokenizer.decode(token_ids, skip_special_tokens=False) |
| 240 | if self.ws_start: |
| 241 | text = decode_whitespaces(text, self.ws_start, self.ws_len) |
| 242 | return text |
| 243 | |
| 244 | @property |
| 245 | def eod(self): |
nothing calls this directly
no test coverage detected