| 152 | |
| 153 | |
| 154 | class FloatQuantizer(BaseQuantizer): |
| 155 | def __init__(self, bit, symmetric, granularity, **kwargs): |
| 156 | super().__init__(bit, symmetric, granularity, **kwargs) |
| 157 | assert self.bit in ["e4m3", "e5m2"], f"Unsupported bit configuration: {self.bit}" |
| 158 | assert self.sym |
| 159 | |
| 160 | if self.bit == "e4m3": |
| 161 | self.e_bits = 4 |
| 162 | self.m_bits = 3 |
| 163 | self.fp_dtype = torch.float8_e4m3fn |
| 164 | elif self.bit == "e5m2": |
| 165 | self.e_bits = 5 |
| 166 | self.m_bits = 2 |
| 167 | self.fp_dtype = torch.float8_e5m2 |
| 168 | else: |
| 169 | raise ValueError(f"Unsupported bit configuration: {self.bit}") |
| 170 | |
| 171 | finfo = torch.finfo(self.fp_dtype) |
| 172 | self.qmin, self.qmax = finfo.min, finfo.max |
| 173 | |
| 174 | self.qmax = torch.tensor(self.qmax) |
| 175 | self.qmin = torch.tensor(self.qmin) |
| 176 | |
| 177 | def quant(self, tensor, scales, zeros, qmax, qmin): |
| 178 | scaled_tensor = tensor / scales + zeros |
| 179 | scaled_tensor = torch.clip(scaled_tensor, self.qmin.cuda(), self.qmax.cuda()) |
| 180 | org_dtype = scaled_tensor.dtype |
| 181 | q_tensor = _load_float_quantize()(scaled_tensor.float(), self.e_bits, self.m_bits, rounding="nearest") |
| 182 | q_tensor.to(org_dtype) |
| 183 | return q_tensor |
| 184 | |
| 185 | def dequant(self, tensor, scales, zeros): |
| 186 | tensor = (tensor - zeros) * scales |
| 187 | return tensor |
| 188 | |
| 189 | def dequant(self, tensor, scales, out_dtype=torch.bfloat16): |
| 190 | tensor_f = tensor.to(torch.float32) |
| 191 | scales_f = scales.to(dtype=torch.float32, device=tensor.device) |
| 192 | out = tensor_f * scales_f |
| 193 | return out.to(out_dtype) |
| 194 | |
| 195 | def quant_dequant(self, tensor, scales, zeros, qmax, qmin): |
| 196 | tensor = self.quant(tensor, scales, zeros, qmax, qmin) |
| 197 | tensor = self.dequant(tensor, scales, zeros) |
| 198 | return tensor |
| 199 | |
| 200 | |
| 201 | def quant_naive_inplace(w, dtype): |
no outgoing calls
no test coverage detected