Check whether a model uses FP8 block-wise quantization by looking at its config.
(config_path: &Path)
| 18 | |
| 19 | /// Check whether a model uses FP8 block-wise quantization by looking at its config. |
| 20 | pub fn is_fp8_quantized(config_path: &Path) -> bool { |
| 21 | let Ok(data) = std::fs::read_to_string(config_path) else { |
| 22 | return false; |
| 23 | }; |
| 24 | let Ok(json) = serde_json::from_str::<serde_json::Value>(&data) else { |
| 25 | return false; |
| 26 | }; |
| 27 | // Check top-level and nested text_config for quantization_config |
| 28 | for root in [&json, json.get("text_config").unwrap_or(&json)] { |
| 29 | let is_fp8 = root |
| 30 | .get("quantization_config") |
| 31 | .and_then(|qc| qc.get("quant_method")) |
| 32 | .and_then(|qm| qm.as_str()) |
| 33 | .map(|s| s == "fp8") |
| 34 | .unwrap_or(false); |
| 35 | if is_fp8 { |
| 36 | return true; |
| 37 | } |
| 38 | } |
| 39 | false |
| 40 | } |
| 41 | |
| 42 | /// Dequantize a 2-D FP8 weight tensor using its per-block scale factor. |
| 43 | pub fn dequantize_fp8_blockwise(weight: &Tensor, scale_inv: &Tensor) -> candle_core::Result<Tensor> { |
no test coverage detected