(self, x: torch.Tensor)
| 607 | destination[prefix + "weight." + k] = v if keep_vars else v.detach() |
| 608 | |
| 609 | def forward(self, x: torch.Tensor): |
| 610 | fix_4bit_weight_quant_state_from_module(self) |
| 611 | quant_state = self.weight.quant_state |
| 612 | |
| 613 | if ( |
| 614 | x.device.type == "cpu" |
| 615 | and self.support_avx512bf16_for_cpu |
| 616 | and not self.training |
| 617 | and x.requires_grad == False |
| 618 | and not getattr(quant_state, "packing_format_for_cpu", False) |
| 619 | ): |
| 620 | self.weight.data, quant_state = _convert_weight_packed_for_cpu(self.weight.data, quant_state) |
| 621 | |
| 622 | if not self.compute_type_is_set: |
| 623 | self.set_compute_type(x) |
| 624 | self.compute_type_is_set = True |
| 625 | |
| 626 | inp_dtype = x.dtype |
| 627 | if self.compute_dtype is not None: |
| 628 | x = x.to(self.compute_dtype) |
| 629 | |
| 630 | bias = self.bias |
| 631 | if bias is not None: |
| 632 | if bias.dtype != x.dtype: |
| 633 | # TODO: do we need to cast bias like this? |
| 634 | bias.data = bias.data.to(x.dtype) |
| 635 | bias = bias.to(self.compute_dtype) |
| 636 | |
| 637 | return bnb.matmul_4bit(x, self.weight, bias=bias, quant_state=quant_state).to(inp_dtype) |
| 638 | |
| 639 | |
| 640 | class LinearFP4(Linear4bit): |
nothing calls this directly
no test coverage detected