| 29 | |
| 30 | |
| 31 | class WeightDecoder(nn.Module): |
| 32 | def __init__( |
| 33 | self, |
| 34 | weight_dim: int = 150, |
| 35 | weight_num: int = 168, |
| 36 | decoder_blocks: int = 4, |
| 37 | add_constant: bool = False, |
| 38 | ): |
| 39 | super(WeightDecoder, self).__init__() |
| 40 | self.weight_num = weight_num |
| 41 | self.weight_dim = weight_dim |
| 42 | |
| 43 | self.register_buffer( |
| 44 | 'block_pos_emb', |
| 45 | _get_sinusoid_encoding_table(weight_num*2, weight_dim) |
| 46 | ) |
| 47 | |
| 48 | # calc heads for mem-eff or flash_attn |
| 49 | heads = 1 |
| 50 | while weight_dim % heads==0 and weight_dim // heads > 64: |
| 51 | heads *= 2 |
| 52 | heads //= 2 |
| 53 | |
| 54 | self.pos_emb_proj = nn.Linear(weight_dim, weight_dim, bias=False) |
| 55 | self.decoder_model = nn.ModuleList( |
| 56 | TransformerBlock(weight_dim, heads, weight_dim//heads, context_dim=weight_dim, gated_ff=False) |
| 57 | for _ in range(decoder_blocks) |
| 58 | ) |
| 59 | # self.delta_proj = nn.Linear(weight_dim, weight_dim, bias=False) |
| 60 | self.delta_proj = nn.Sequential( |
| 61 | nn.LayerNorm(weight_dim), |
| 62 | nn.Linear(weight_dim, weight_dim, bias=False) |
| 63 | ) |
| 64 | self.init_weights(add_constant) |
| 65 | |
| 66 | def init_weights(self, add_constant: bool = False): |
| 67 | def basic_init(module): |
| 68 | if isinstance(module, nn.Linear): |
| 69 | nn.init.xavier_uniform_(module.weight) |
| 70 | if module.bias is not None: |
| 71 | nn.init.constant_(module.bias, 0) |
| 72 | self.apply(basic_init) |
| 73 | |
| 74 | # For no pre-optimized training, you should consider use the following init |
| 75 | # with self.down = down@down_aux + 1 in LiLoRAAttnProcessor |
| 76 | # if add_constant: |
| 77 | torch.nn.init.constant_(self.delta_proj[1].weight, 0) |
| 78 | |
| 79 | # advice from Nataniel Ruiz, looks like 1e-3 is small enough |
| 80 | # else: |
| 81 | # torch.nn.init.normal_(self.delta_proj[1].weight, std=1e-3) |
| 82 | |
| 83 | def forward(self, weight, features): |
| 84 | pos_emb = self.pos_emb_proj(self.block_pos_emb[:, :weight.size(1)].clone().detach()) |
| 85 | h = weight + pos_emb |
| 86 | for decoder in self.decoder_model: |
| 87 | h = decoder(h, context=features) |
| 88 | weight = weight + self.delta_proj(h) |