| 294 | auto c_gate_msa = slice_conditioning_plane(ctx, modulation, 5, hidden_size, seq_len); |
| 295 | |
| 296 | auto norm_hidden = apply_modulated_rms_norm( |
| 297 | ctx, |
| 298 | hidden_states, |
| 299 | weights.self_attn_norm, |
| 300 | one, |
| 301 | shift_msa, |
| 302 | scale_msa, |
| 303 | config.rms_norm_eps); |
| 304 | auto attn_out = build_attention( |
| 305 | ctx, |
| 306 | norm_hidden, |
| 307 | positions, |
| 308 | weights.self_attn, |
| 309 | config, |
| 310 | self_attention_mask, |
| 311 | backend_type); |
| 312 | attn_out = modules::MulModule{}.build(ctx, attn_out, gate_msa); |
| 313 | auto x = modules::AddModule{}.build(ctx, hidden_states, attn_out); |
| 314 |
no test coverage detected