| 166 | diff_w, diff_b = self.get_diff_weight(scale, device=device) |
| 167 | weight = self.org_weight + diff_w * drop |
| 168 | if self.org_bias is not None: |
| 169 | bias = self.org_bias + diff_b * drop |
| 170 | else: |
| 171 | bias = None |
| 172 | else: |
| 173 | weight = self.weight |
| 174 | bias = self.bias |
| 175 | return weight, bias |
| 176 | |
| 177 | def get_diff_weight(self, multiplier=1, shape=None, device=None): |
| 178 | if self.is_diff: |
| 179 | diff_b = None |
| 180 | if self.bias is not None: |
| 181 | diff_b = self.bias * multiplier |
| 182 | return self.weight * multiplier, diff_b |
| 183 | org_weight = self.org_module[0].weight.to(device, dtype=self.weight.dtype) |
| 184 | diff = self.weight.to(device) - org_weight |
| 185 | diff_b = None |
| 186 | if shape: |
| 187 | diff = diff.view(shape) |
| 188 | if self.bias is not None: |
| 189 | org_bias = self.org_module[0].bias.to(device, dtype=self.bias.dtype) |
| 190 | diff_b = self.bias.to(device) - org_bias |
| 191 | if device is not None: |
| 192 | diff = diff.to(device) |
| 193 | if self.bias is not None: |