Read an expert weight, handling GPTQ dequantization if needed.
(&self, weight_name: &str)
| 379 | |
| 380 | /// Read an expert weight, handling GPTQ dequantization if needed. |
| 381 | fn read_expert_weight(&self, weight_name: &str) -> Result<Tensor> { |
| 382 | if let Some(group_size) = self.gptq_group_size { |
| 383 | let prefix = weight_name.strip_suffix(".weight").unwrap_or(weight_name); |
| 384 | let qw_name = format!("{prefix}.qweight"); |
| 385 | if self.storage.has_tensor(&qw_name) { |
| 386 | let sc_name = format!("{prefix}.scales"); |
| 387 | let qz_name = format!("{prefix}.qzeros"); |
| 388 | // Read the GPTQ triplet |
| 389 | let qw_data = self.storage.read_tensor(&qw_name) |
| 390 | .map_err(|e| candle_core::Error::Msg(format!("read qweight: {e}")))?; |
| 391 | let sc_data = self.storage.read_tensor(&sc_name) |
| 392 | .map_err(|e| candle_core::Error::Msg(format!("read scales: {e}")))?; |
| 393 | let qz_data = self.storage.read_tensor(&qz_name) |
| 394 | .map_err(|e| candle_core::Error::Msg(format!("read qzeros: {e}")))?; |
| 395 | // Materialize to CPU tensors |
| 396 | let qw = Tensor::from_raw_buffer(&qw_data.bytes, qw_data.dtype, &qw_data.shape, &Device::Cpu)?; |
| 397 | let sc = Tensor::from_raw_buffer(&sc_data.bytes, sc_data.dtype, &sc_data.shape, &Device::Cpu)?; |
| 398 | let qz = Tensor::from_raw_buffer(&qz_data.bytes, qz_data.dtype, &qz_data.shape, &Device::Cpu)?; |
| 399 | // Dequantize |
| 400 | let weight = crate::utils::gptq::dequantize_gptq_4bit(&qw, &sc, &qz, group_size)?; |
| 401 | let weight = weight.to_dtype(self.dtype)?; |
| 402 | return if self.needs_device_transfer { |
| 403 | weight.to_device(&self.device) |
| 404 | } else { |
| 405 | Ok(weight) |
| 406 | }; |
| 407 | } |
| 408 | } |
| 409 | // Non-GPTQ: read plain weight tensor |
| 410 | let data = self.storage.read_tensor(weight_name) |
| 411 | .map_err(|e| candle_core::Error::Msg(format!("read_tensor: {e}")))?; |
| 412 | self.materialize(data) |
| 413 | } |
| 414 | |
| 415 | /// Convert raw TensorData to a candle Tensor with target dtype/device. |
| 416 | #[inline] |
no test coverage detected