| 1783 | layer.norm_mha = load_flow_layer_norm(*weights->store, source, prefix + ".norm_mha", 512); |
| 1784 | layer.attn.q = load_flow_linear(*weights->store, source, prefix + ".self_attn.linear_q", 512, 512, true, weight_storage_type); |
| 1785 | layer.attn.k = load_flow_linear(*weights->store, source, prefix + ".self_attn.linear_k", 512, 512, true, weight_storage_type); |
| 1786 | layer.attn.v = load_flow_linear(*weights->store, source, prefix + ".self_attn.linear_v", 512, 512, true, weight_storage_type); |
| 1787 | layer.attn.out = load_flow_linear(*weights->store, source, prefix + ".self_attn.linear_out", 512, 512, true, weight_storage_type); |
| 1788 | layer.attn.pos = load_flow_linear(*weights->store, source, prefix + ".self_attn.linear_pos", 512, 512, false, weight_storage_type); |
| 1789 | layer.attn.pos_bias_u_tensor = weights->store->load_f32_tensor(source, prefix + ".self_attn.pos_bias_u", {8, 64}); |
| 1790 | layer.attn.pos_bias_v_tensor = weights->store->load_f32_tensor(source, prefix + ".self_attn.pos_bias_v", {8, 64}); |
| 1791 | layer.norm_ff = load_flow_layer_norm(*weights->store, source, prefix + ".norm_ff", 512); |
| 1792 | layer.ff.w1 = load_flow_linear(*weights->store, source, prefix + ".feed_forward.w_1", 2048, 512, true, weight_storage_type); |
| 1793 | layer.ff.w2 = load_flow_linear(*weights->store, source, prefix + ".feed_forward.w_2", 512, 2048, true, weight_storage_type); |
| 1794 | return layer; |
| 1795 | }; |
| 1796 | for (int i = 0; i < 6; ++i) { |
| 1797 | weights->encoders.push_back(load_layer("flow.encoder.encoders." + std::to_string(i))); |
no test coverage detected