Interface Quantization

cake-core/src/utils/mod.rs:32–57

A quantization strategy that can create a VarBuilder with appropriate dequantization and estimate in-memory VRAM from on-disk size. Implementations: `NoQuantization`, `Fp8Quantization`, `GptqQuantization`.

///
/// Implementations: `NoQuantization`, `Fp8Quantization`, `GptqQuantization`.
pub trait Quantization: Send + Sync {
    /// Human-readable name for logging.
    fn name(&self) -> &str;

    /// Create a VarBuilder from safetensors files with this quantization.
    ///
    /// # Safety
    /// Inherits the mmap safety requirements from candle's safetensors loading.
    unsafe fn load_var_builder<'a>(
        &self,
        filenames: &[PathBuf],
        dtype: DType,
        device: &Device,
    ) -> Result<VarBuilder<'a>>;

    /// Returns the GPTQ group size if this is GPTQ quantization, None otherwise.
    fn gptq_group_size(&self) -> Option<usize> {
        None
    }

    /// Estimate in-memory layer size given on-disk size and target dtype bytes.
    /// Default: no expansion (on-disk size = in-memory size).
    fn estimate_layer_vram(&self, on_disk_bytes: u64, _dtype_bytes: u64) -> u64 {
        on_disk_bytes
    }
}

/// No quantization — plain safetensors, loaded as-is.
pub struct NoQuantization;