(t)
| 38 | return self.tokenizer(t, truncation=False, add_special_tokens=False)["input_ids"] |
| 39 | |
| 40 | def transformer_encode(t): |
| 41 | if self.clip_skip > 1: |
| 42 | rt = self.transformer(input_ids=t, output_hidden_states=True) |
| 43 | return self.transformer.text_model.final_layer_norm(rt.hidden_states[-self.clip_skip]) |
| 44 | else: |
| 45 | return self.transformer(input_ids=t, output_hidden_states=False).last_hidden_state |
| 46 | |
| 47 | def split(x): |
| 48 | return x[75 * 0: 75 * 1], x[75 * 1: 75 * 2], x[75 * 2: 75 * 3] |
no outgoing calls
no test coverage detected