(self, x, *args, **kwargs)
| 79 | del self.org_module |
| 80 | |
| 81 | def forward(self, x, *args, **kwargs): |
| 82 | weight_dtype = x.dtype |
| 83 | org_forwarded = self.org_forward(x) |
| 84 | |
| 85 | # module dropout |
| 86 | if self.module_dropout is not None and self.training: |
| 87 | if torch.rand(1) < self.module_dropout: |
| 88 | return org_forwarded |
| 89 | |
| 90 | lx = self.lora_down(x.to(self.lora_down.weight.dtype)) |
| 91 | |
| 92 | # normal dropout |
| 93 | if self.dropout is not None and self.training: |
| 94 | lx = torch.nn.functional.dropout(lx, p=self.dropout) |
| 95 | |
| 96 | # rank dropout |
| 97 | if self.rank_dropout is not None and self.training: |
| 98 | mask = torch.rand((lx.size(0), self.lora_dim), device=lx.device) > self.rank_dropout |
| 99 | if len(lx.size()) == 3: |
| 100 | mask = mask.unsqueeze(1) # for Text Encoder |
| 101 | elif len(lx.size()) == 4: |
| 102 | mask = mask.unsqueeze(-1).unsqueeze(-1) # for Conv2d |
| 103 | lx = lx * mask |
| 104 | |
| 105 | # scaling for rank dropout: treat as if the rank is changed |
| 106 | scale = self.scale * (1.0 / (1.0 - self.rank_dropout)) # redundant for readability |
| 107 | else: |
| 108 | scale = self.scale |
| 109 | |
| 110 | lx = self.lora_up(lx) |
| 111 | |
| 112 | return org_forwarded.to(weight_dtype) + lx.to(weight_dtype) * self.multiplier * scale |
| 113 | |
| 114 | |
| 115 | def addnet_hash_legacy(b): |
nothing calls this directly
no outgoing calls
no test coverage detected