| 311 | backend_type); |
| 312 | attn_out = modules::MulModule{}.build(ctx, attn_out, gate_msa); |
| 313 | auto x = modules::AddModule{}.build(ctx, hidden_states, attn_out); |
| 314 | |
| 315 | auto cross_norm = modules::RMSNormModule({hidden_size, config.rms_norm_eps, true, false}) |
| 316 | .build(ctx, x, {weights.cross_attn_norm, std::nullopt}); |
| 317 | auto cross_out = build_attention( |
| 318 | ctx, |
| 319 | cross_norm, |
| 320 | positions, |
| 321 | weights.cross_attn, |
| 322 | config, |
| 323 | encoder_attention_mask, |
| 324 | backend_type, |
| 325 | cross_attention_kv, |
| 326 | encoder_hidden_states); |
| 327 | x = modules::AddModule{}.build(ctx, x, cross_out); |
| 328 | |
| 329 | auto mlp_norm = apply_modulated_rms_norm( |
no test coverage detected