(
self,
weight_dim: int = 150,
weight_num: int = 168,
decoder_blocks: int = 4,
add_constant: bool = False,
)
| 30 | |
| 31 | class WeightDecoder(nn.Module): |
| 32 | def __init__( |
| 33 | self, |
| 34 | weight_dim: int = 150, |
| 35 | weight_num: int = 168, |
| 36 | decoder_blocks: int = 4, |
| 37 | add_constant: bool = False, |
| 38 | ): |
| 39 | super(WeightDecoder, self).__init__() |
| 40 | self.weight_num = weight_num |
| 41 | self.weight_dim = weight_dim |
| 42 | |
| 43 | self.register_buffer( |
| 44 | 'block_pos_emb', |
| 45 | _get_sinusoid_encoding_table(weight_num*2, weight_dim) |
| 46 | ) |
| 47 | |
| 48 | # calc heads for mem-eff or flash_attn |
| 49 | heads = 1 |
| 50 | while weight_dim % heads==0 and weight_dim // heads > 64: |
| 51 | heads *= 2 |
| 52 | heads //= 2 |
| 53 | |
| 54 | self.pos_emb_proj = nn.Linear(weight_dim, weight_dim, bias=False) |
| 55 | self.decoder_model = nn.ModuleList( |
| 56 | TransformerBlock(weight_dim, heads, weight_dim//heads, context_dim=weight_dim, gated_ff=False) |
| 57 | for _ in range(decoder_blocks) |
| 58 | ) |
| 59 | # self.delta_proj = nn.Linear(weight_dim, weight_dim, bias=False) |
| 60 | self.delta_proj = nn.Sequential( |
| 61 | nn.LayerNorm(weight_dim), |
| 62 | nn.Linear(weight_dim, weight_dim, bias=False) |
| 63 | ) |
| 64 | self.init_weights(add_constant) |
| 65 | |
| 66 | def init_weights(self, add_constant: bool = False): |
| 67 | def basic_init(module): |
nothing calls this directly
no test coverage detected