Initialize Linear8bitLt class. Args: input_features (`int`): Number of input features of the linear layer. output_features (`int`): Number of output features of the linear layer. bias (`bool`, defaults to `True`):
(
self,
input_features: int,
output_features: int,
bias=True,
has_fp16_weights=True,
threshold=0.0,
index=None,
device=None,
)
| 1048 | """ |
| 1049 | |
| 1050 | def __init__( |
| 1051 | self, |
| 1052 | input_features: int, |
| 1053 | output_features: int, |
| 1054 | bias=True, |
| 1055 | has_fp16_weights=True, |
| 1056 | threshold=0.0, |
| 1057 | index=None, |
| 1058 | device=None, |
| 1059 | ): |
| 1060 | """ |
| 1061 | Initialize Linear8bitLt class. |
| 1062 | |
| 1063 | Args: |
| 1064 | input_features (`int`): |
| 1065 | Number of input features of the linear layer. |
| 1066 | output_features (`int`): |
| 1067 | Number of output features of the linear layer. |
| 1068 | bias (`bool`, defaults to `True`): |
| 1069 | Whether the linear class uses the bias term as well. |
| 1070 | has_fp16_weights (`bool`, defaults to `True`): |
| 1071 | If False, weights are quantized to int8 on ``.to(device)``. If True, |
| 1072 | weights remain in fp16 and are quantized on-the-fly during each forward pass. |
| 1073 | threshold (`float`, defaults to `0.0`): |
| 1074 | Outlier threshold for mixed-precision decomposition (LLM.int8()). During the |
| 1075 | forward pass, activation columns where any value exceeds this threshold are |
| 1076 | computed in fp16, while the remaining columns use int8. This operates on |
| 1077 | **activations** (inputs), not on weight values. Set to 0.0 to disable |
| 1078 | mixed-precision decomposition and quantize all columns to int8. |
| 1079 | index: Indices for weight reordering (used internally). |
| 1080 | device: Device to initialize the layer on. |
| 1081 | """ |
| 1082 | super().__init__(input_features, output_features, bias, device) |
| 1083 | self.state = bnb.MatmulLtState() |
| 1084 | self.index = index |
| 1085 | |
| 1086 | self.state.threshold = threshold |
| 1087 | self.state.has_fp16_weights = has_fp16_weights |
| 1088 | |
| 1089 | if threshold > 0.0 and not has_fp16_weights: |
| 1090 | self.state.use_pool = True |
| 1091 | |
| 1092 | self.weight = Int8Params(self.weight.data, has_fp16_weights=has_fp16_weights, requires_grad=has_fp16_weights) |
| 1093 | self._register_load_state_dict_pre_hook(maybe_rearrange_weight) |
| 1094 | |
| 1095 | def _save_to_state_dict(self, destination, prefix, keep_vars): |
| 1096 | super()._save_to_state_dict(destination, prefix, keep_vars) |
nothing calls this directly
no test coverage detected