| 404 | return attn_output |
| 405 | |
| 406 | class CLIPEncoderLayer: |
| 407 | def __init__(self): |
| 408 | self.self_attn = CLIPAttention() |
| 409 | self.layer_norm1 = LayerNorm(768) |
| 410 | self.mlp = CLIPMLP() |
| 411 | self.layer_norm2 = LayerNorm(768) |
| 412 | |
| 413 | def __call__(self, hidden_states, causal_attention_mask): |
| 414 | residual = hidden_states |
| 415 | hidden_states = self.layer_norm1(hidden_states) |
| 416 | hidden_states = self.self_attn(hidden_states, causal_attention_mask) |
| 417 | hidden_states = residual + hidden_states |
| 418 | |
| 419 | residual = hidden_states |
| 420 | hidden_states = self.layer_norm2(hidden_states) |
| 421 | hidden_states = self.mlp(hidden_states) |
| 422 | hidden_states = residual + hidden_states |
| 423 | |
| 424 | return hidden_states |
| 425 | |
| 426 | class CLIPEncoder: |
| 427 | def __init__(self): |