Pre-dequantize F8→F16 and cache. Call once before inference loop.
(&mut self)
| 221 | |
| 222 | /// Pre-dequantize F8→F16 and cache. Call once before inference loop. |
| 223 | pub fn warmup(&mut self) -> candle_core::Result<()> { |
| 224 | let w = self.get_weight()?; |
| 225 | *self.weight.write().unwrap() = Some(w); |
| 226 | self.f8_weight = None; // Free the F8 storage |
| 227 | Ok(()) |
| 228 | } |
| 229 | |
| 230 | /// Forward pass: dequantizes weight if needed, computes matmul in weight dtype. |
| 231 | /// The cached weight is already transposed and contiguous for optimal matmul. |