| 350 | return x.sequential(self.out) |
| 351 | |
| 352 | class CLIPMLP: |
| 353 | def __init__(self): |
| 354 | self.fc1 = Linear(768, 3072) |
| 355 | self.fc2 = Linear(3072, 768) |
| 356 | |
| 357 | def __call__(self, hidden_states): |
| 358 | hidden_states = self.fc1(hidden_states) |
| 359 | hidden_states = hidden_states.quick_gelu() |
| 360 | hidden_states = self.fc2(hidden_states) |
| 361 | return hidden_states |
| 362 | |
| 363 | class CLIPAttention: |
| 364 | def __init__(self): |