(t)
| 35 | BOS = self.tokenizer.bos_token_id |
| 36 | |
| 37 | def tokenize(t): |
| 38 | return self.tokenizer(t, truncation=False, add_special_tokens=False)["input_ids"] |
| 39 | |
| 40 | def transformer_encode(t): |
| 41 | if self.clip_skip > 1: |
no outgoing calls
no test coverage detected