(
&self,
name: &str,
dtype: DType,
dev: &Device,
)
| 210 | |
| 211 | impl GptqBackend { |
| 212 | fn load_tensor( |
| 213 | &self, |
| 214 | name: &str, |
| 215 | dtype: DType, |
| 216 | dev: &Device, |
| 217 | ) -> candle_core::Result<Tensor> { |
| 218 | // Strip the ".weight" suffix to get the parameter prefix. |
| 219 | let prefix = name.strip_suffix(".weight").unwrap_or(name); |
| 220 | let qweight_name = format!("{prefix}.qweight"); |
| 221 | let scales_name = format!("{prefix}.scales"); |
| 222 | let qzeros_name = format!("{prefix}.qzeros"); |
| 223 | |
| 224 | if self.inner.get(&qweight_name).is_ok() { |
| 225 | // Standard GPTQ: qweight + scales + qzeros |
| 226 | let qweight = self.inner.load(&qweight_name, &Device::Cpu)?; |
| 227 | let scales = self.inner.load(&scales_name, &Device::Cpu)?; |
| 228 | let qzeros = self.inner.load(&qzeros_name, &Device::Cpu)?; |
| 229 | let weight = dequantize_gptq_4bit(&qweight, &scales, &qzeros, self.group_size)?; |
| 230 | weight.to_dtype(dtype)?.to_device(dev) |
| 231 | } else if self.inner.get(&scales_name).is_ok() { |
| 232 | // Affine 4-bit quantization: packed uint32 weight + scales + biases |
| 233 | // Formula: w4 * scale + bias (no zero-point) |
| 234 | let biases_name = format!("{prefix}.biases"); |
| 235 | let packed = self.inner.load(name, &Device::Cpu)?; |
| 236 | let scales = self.inner.load(&scales_name, &Device::Cpu)?; |
| 237 | let biases = self.inner.load(&biases_name, &Device::Cpu)?; |
| 238 | let weight = dequantize_packed_4bit(&packed, &scales, &biases, self.group_size)?; |
| 239 | weight.to_dtype(dtype)?.to_device(dev) |
| 240 | } else { |
| 241 | // Non-quantized tensor — load directly. |
| 242 | self.inner.load(name, dev)?.to_dtype(dtype) |
| 243 | } |
| 244 | } |
| 245 | } |
| 246 | |
| 247 | impl SimpleBackend for GptqBackend { |
no test coverage detected