(self, x)
| 56 | self.register_buffer('bias', torch.empty(out_features)) |
| 57 | |
| 58 | def forward(self, x): |
| 59 | if self.bits in [2, 4, 8, 16]: |
| 60 | weight_unpack = torch.bitwise_right_shift(torch.unsqueeze(self.qweight, 1).expand(-1, 32 // self.bits, -1), self.wf.unsqueeze(-1)).to(torch.int16 if self.bits == 8 else torch.int8).view(-1, self.qweight.size(-1)) |
| 61 | torch.bitwise_and(weight_unpack,(2 ** self.bits) - 1, out=weight_unpack) |
| 62 | else: |
| 63 | raise ValueError |
| 64 | |
| 65 | if self.asym: |
| 66 | zeros_unpack = torch.bitwise_right_shift(torch.unsqueeze(self.qzeros, 2).expand(-1, -1, 32 // self.bits), self.wf.unsqueeze(0)).to(torch.int16 if self.bits == 8 else torch.int8) |
| 67 | torch.bitwise_and(zeros_unpack, (2 ** self.bits) - 1, out=zeros_unpack) |
| 68 | |
| 69 | zeros_unpack = zeros_unpack + 1 |
| 70 | zeros_unpack = zeros_unpack.reshape(-1, self.out_features) |
| 71 | zeros_unpack = zeros_unpack[self.g_idx.long()] |
| 72 | |
| 73 | if self.bits == 2: |
| 74 | qscales = self.qscales.unsqueeze(-1) |
| 75 | else: |
| 76 | qscales = self.qscales |
| 77 | |
| 78 | scales = ((qscales.to(x.dtype)-self.qscales_zeros)*self.qscales_scales).view(math.ceil(self.in_features/self.groupsize), self.out_features)[self.g_idx.long()] |
| 79 | |
| 80 | weight = ((weight_unpack - zeros_unpack)*scales).type(x.dtype) |
| 81 | |
| 82 | else: |
| 83 | qstatistic = self.qstatistic.to(torch.uint8) |
| 84 | qscales = (qstatistic & 0xF0) >> 4 |
| 85 | qzeros = qstatistic & 0x0F |
| 86 | |
| 87 | scales = ((qscales.to(x.dtype)-self.qscales_zeros)*self.qscales_scales).view(math.ceil(self.in_features/self.groupsize), self.out_features)[self.g_idx.long()] |
| 88 | zeros = ((qzeros.to(x.dtype)-self.qzeros_zeros)*self.qzeros_scales).view(math.ceil(self.in_features/self.groupsize), self.out_features)[self.g_idx.long()] |
| 89 | |
| 90 | weight = (weight_unpack*scales-zeros).type(x.dtype) |
| 91 | |
| 92 | out = torch.matmul(x, weight) |
| 93 | |
| 94 | if not self.disable_bias: |
| 95 | out += self.bias |
| 96 | return out |
| 97 | |
| 98 | |
| 99 | def make_quant(module, names, name='', groupsize=-1, double_groupsize=-1, bits=4, v1=True, asym=True): |
nothing calls this directly
no outgoing calls
no test coverage detected