(
pt2e_quantize: str,
quantization_mode: Optional[str] = None,
is_qat: bool = False,
)
| 141 | |
| 142 | |
| 143 | def get_qnn_quantizer( |
| 144 | pt2e_quantize: str, |
| 145 | quantization_mode: Optional[str] = None, |
| 146 | is_qat: bool = False, |
| 147 | ): |
| 148 | try: |
| 149 | from executorch.backends.qualcomm.quantizer.custom_annotation import ( # pyre-fixme[21] |
| 150 | custom_annotate_llama_matmul_16a8w, |
| 151 | ) |
| 152 | |
| 153 | # pyre-ignore: Undefined import [21]: Could not find a module corresponding to import `executorch.backends.qualcomm.quantizer.quantizer` |
| 154 | from executorch.backends.qualcomm.quantizer.quantizer import ( |
| 155 | QnnQuantizer, |
| 156 | QuantDtype, |
| 157 | ) |
| 158 | from torchao.quantization.pt2e import MinMaxObserver |
| 159 | |
| 160 | except ImportError: |
| 161 | raise ImportError( |
| 162 | "Please install the Qualcomm backend follwing https://pytorch.org/executorch/main/backends-qualcomm" |
| 163 | ) |
| 164 | |
| 165 | backend, quant_config = pt2e_quantize.split("_") |
| 166 | assert ( |
| 167 | backend == "qnn" |
| 168 | ), f"The quantization config is for backend {backend} instead of qnn." |
| 169 | qnn_quantizer = QnnQuantizer() # pyre-fixme[16] |
| 170 | |
| 171 | # more custom quantization are supported including 16a4w etc. default to 8bit quantized |
| 172 | custom_annotations = () |
| 173 | if quant_config == "8a8w": |
| 174 | quant_dtype = QuantDtype.use_8a8w # pyre-fixme[16] |
| 175 | qnn_quantizer.set_default_quant_config( |
| 176 | quant_dtype, |
| 177 | is_qat=is_qat, |
| 178 | is_conv_per_channel=True, |
| 179 | is_linear_per_channel=True, |
| 180 | ) |
| 181 | elif quant_config == "16a16w": |
| 182 | # Due to the error with 16a16w in Qnn Htp, we need to disable per channel linear quantization when use 16a16w |
| 183 | # TODO: enable it after the issue is fixed |
| 184 | logging.warning( |
| 185 | "Disable per channel quantization for linear and conv due to the error with QNN HTP 16a16w." |
| 186 | ) |
| 187 | quant_dtype = QuantDtype.use_16a16w # pyre-fixme[16] |
| 188 | qnn_quantizer.set_default_quant_config( |
| 189 | quant_dtype, |
| 190 | is_qat=is_qat, |
| 191 | is_conv_per_channel=False, |
| 192 | is_linear_per_channel=False, |
| 193 | act_observer=MinMaxObserver, |
| 194 | ) |
| 195 | elif quant_config == "16a4w": |
| 196 | quant_dtype = QuantDtype.use_16a4w # pyre-fixme[16] |
| 197 | qnn_quantizer.set_default_quant_config( |
| 198 | quant_dtype, |
| 199 | is_qat=is_qat, |
| 200 | is_conv_per_channel=True, |
no test coverage detected