Load this block from the VarBuilder given the specific configuration.
(vb: VarBuilder, cfg: &super::Config, backend: Arc<dyn ComputeBackend>)
| 32 | |
| 33 | /// Load this block from the VarBuilder given the specific configuration. |
| 34 | pub fn load(vb: VarBuilder, cfg: &super::Config, backend: Arc<dyn ComputeBackend>) -> Result<Self> { |
| 35 | let h_size = cfg.hidden_size; |
| 36 | let i_size = cfg.intermediate_size; |
| 37 | |
| 38 | let gate_up_w = if cfg.fused_gate_up_proj { |
| 39 | // Phi-3/4 style: weights already fused as 'gate_up_proj' |
| 40 | vb.pp("gate_up_proj").get((2 * i_size, h_size), "weight")? |
| 41 | } else { |
| 42 | // Standard: fuse gate_proj and up_proj into a single matmul |
| 43 | let gate_w = vb.pp("gate_proj").get((i_size, h_size), "weight")?; |
| 44 | let up_w = vb.pp("up_proj").get((i_size, h_size), "weight")?; |
| 45 | Tensor::cat(&[&gate_w, &up_w], 0)? |
| 46 | }; |
| 47 | let gate_up_proj_weight = backend.preprocess_linear_weight(&gate_up_w)?; |
| 48 | |
| 49 | let down_w = vb.pp("down_proj").get((h_size, i_size), "weight")?; |
| 50 | let down_proj_weight = backend.preprocess_linear_weight(&down_w)?; |
| 51 | |
| 52 | Ok(Self { |
| 53 | gate_up_proj_weight, |
| 54 | down_proj_weight, |
| 55 | intermediate_size: i_size, |
| 56 | use_gelu: cfg.use_gelu_mlp, |
| 57 | backend, |
| 58 | }) |
| 59 | } |
| 60 | } |
nothing calls this directly
no test coverage detected