(self, scale=1, device=None)
| 148 | diff_bias = state_dict.pop(f"{prefix}diff_b") |
| 149 | state_dict[f"{prefix}bias"] = diff_bias + self.bias.data.to(diff_bias) |
| 150 | |
| 151 | def make_weight(self, scale=1, device=None): |
| 152 | dropping = bool(self.rank_dropout) and self.training |
| 153 | if self.is_diff and not dropping: |
| 154 | # self.weight IS the diff here, so the merge is one scaled add. |
| 155 | weight = add_scaled(self.org_weight.to(device), self.weight, scale) |
| 156 | bias = None |
| 157 | if self.bias is not None and self.org_bias is not None: |
| 158 | # org_bias is cached as a one-element list, like _org_weight. |
| 159 | bias = add_scaled(self.org_bias[0].to(device), self.bias, scale) |
| 160 | return weight, bias |
| 161 | |
| 162 | drop = ( |
| 163 | torch.rand(self.dim, device=device) > self.rank_dropout if dropping else 1 |
| 164 | ) |
| 165 | if drop != 1 or scale != 1 or self.is_diff: |
| 166 | diff_w, diff_b = self.get_diff_weight(scale, device=device) |
| 167 | weight = self.org_weight + diff_w * drop |
| 168 | if self.org_bias is not None: |
| 169 | bias = self.org_bias + diff_b * drop |
no test coverage detected