| 108 | return fp_quant_module.get_scales(self.scale, self.num_groups) |
| 109 | |
| 110 | def dequantize(self, input_q, fp_out=None, q_bits=8, q_mantisa_bits=3, scale=None) -> torch.Tensor: |
| 111 | assert (self.orig_dtype is not None), \ |
| 112 | "[De-quantization Error]: you need to call quantize before dequantizing!" |
| 113 | fp_out = torch.empty(self.orig_shape, dtype=self.orig_dtype, |
| 114 | device=input_q.device) if fp_out is None else fp_out |
| 115 | if q_bits == 8: |
| 116 | pass |
| 117 | elif q_bits == 12: |
| 118 | q_mantisa_bits = 4 |
| 119 | elif q_bits == 6: |
| 120 | q_mantisa_bits = 2 |
| 121 | elif q_bits == 4: |
| 122 | q_mantisa_bits = 1 |
| 123 | else: |
| 124 | assert (0), \ |
| 125 | f"Missing {q_bits}-dequantization, please add the template arguments for the kernel to support this precision!" |
| 126 | |
| 127 | if scale is not None: |
| 128 | assert input_q.numel() == fp_out.numel(), \ |
| 129 | '[De-quantization Error]: quantized data should have the same size as original tensor when scale is not None!' |
| 130 | input_q = torch.cat([input_q.reshape(-1, self.group_size), scale], dim=-1).contiguous() |
| 131 | fp_quant_module.dequantize(fp_out, input_q, self.group_size, q_mantisa_bits, q_bits - q_mantisa_bits - 1) |
| 132 | return fp_out |
| 133 | |
| 134 | def selective_dequantize(self, |
| 135 | input_q, |