| 135 | self.tokenizer_mode = tokenizer_mode |
| 136 | |
| 137 | def _verify_quantization(self) -> None: |
| 138 | supported_quantization = ["awq", "gptq", "squeezellm"] |
| 139 | rocm_not_supported_quantization = ["awq"] |
| 140 | if self.quantization is not None: |
| 141 | self.quantization = self.quantization.lower() |
| 142 | |
| 143 | # Parse quantization method from the HF model config, if available. |
| 144 | hf_quant_config = getattr(self.hf_config, "quantization_config", None) |
| 145 | if hf_quant_config is not None: |
| 146 | hf_quant_method = str(hf_quant_config["quant_method"]).lower() |
| 147 | if self.quantization is None: |
| 148 | self.quantization = hf_quant_method |
| 149 | elif self.quantization != hf_quant_method: |
| 150 | raise ValueError( |
| 151 | "Quantization method specified in the model config " |
| 152 | f"({hf_quant_method}) does not match the quantization " |
| 153 | f"method specified in the `quantization` argument " |
| 154 | f"({self.quantization}).") |
| 155 | |
| 156 | if self.quantization is not None: |
| 157 | if self.quantization not in supported_quantization: |
| 158 | raise ValueError( |
| 159 | f"Unknown quantization method: {self.quantization}. Must " |
| 160 | f"be one of {supported_quantization}.") |
| 161 | if is_hip( |
| 162 | ) and self.quantization in rocm_not_supported_quantization: |
| 163 | raise ValueError( |
| 164 | f"{self.quantization} quantization is currently not supported " |
| 165 | f"in ROCm.") |
| 166 | logger.warning(f"{self.quantization} quantization is not fully " |
| 167 | "optimized yet. The speed can be slower than " |
| 168 | "non-quantized models.") |
| 169 | |
| 170 | def verify_with_parallel_config( |
| 171 | self, |