| 202 | } |
| 203 | |
| 204 | fn get_weight(&self) -> candle_core::Result<candle_core::Tensor> { |
| 205 | // Fast path: return cached dequantized + transposed weight |
| 206 | if let Some(w) = self.weight.read().unwrap().as_ref() { |
| 207 | return Ok(w.clone()); |
| 208 | } |
| 209 | // Slow path: dequantize F8→F32→F16, transpose, and cache |
| 210 | let f8_w = self |
| 211 | .f8_weight |
| 212 | .as_ref() |
| 213 | .expect("no F8 weight and no cached weight"); |
| 214 | // Pre-transpose and contiguous so forward() avoids per-call transpose |
| 215 | let dequantized = f8_w.to_dtype(candle_core::DType::F32)? |
| 216 | .to_dtype(candle_core::DType::F16)? |
| 217 | .t()?.contiguous()?; |
| 218 | *self.weight.write().unwrap() = Some(dequantized.clone()); |
| 219 | Ok(dequantized) |
| 220 | } |
| 221 | |
| 222 | /// Pre-dequantize F8→F16 and cache. Call once before inference loop. |
| 223 | pub fn warmup(&mut self) -> candle_core::Result<()> { |