| 284 | drop = drop.view(1, -1, 1, 1) |
| 285 | else: |
| 286 | drop = drop.view(*[1] * (dims - 1), -1) |
| 287 | mid = mid * drop |
| 288 | if self.module_type == "linear": |
| 289 | output = F.linear(mid, self.lora_up.weight.to(mid)) |
| 290 | else: |
| 291 | output = self.lora_up(mid) |
| 292 | scalar = self.scalar.to(device=output.device, dtype=output.dtype) |
| 293 | return self.dropout(output * scalar * self.scale * scale) |
| 294 | |
| 295 | # Geometry from the conv that carries it, not the org module: neither |
| 296 | # lora conv is built with the org dilation or groups. |
| 297 | op = self.lora_mid if self.tucker else self.lora_down |
| 298 | extra_args = ( |
| 299 | { |
| 300 | "stride": op.stride, |
| 301 | "padding": op.padding, |
| 302 | "dilation": op.dilation, |
| 303 | "groups": op.groups, |
| 304 | } |
| 305 | if self.isconv |
| 306 | else {} |
| 307 | ) |