(self, config: Config, block_idx: int)
| 127 | we replace the attention layer where adaption is implemented.""" |
| 128 | |
| 129 | def __init__(self, config: Config, block_idx: int) -> None: |
| 130 | super().__init__() |
| 131 | self.norm_1 = config.norm_class(config.n_embd, eps=config.norm_eps) |
| 132 | self.attn = CausalSelfAttention(config, block_idx) |
| 133 | if not config.shared_attention_norm: |
| 134 | self.norm_2 = config.norm_class(config.n_embd, eps=config.norm_eps) |
| 135 | self.mlp = config.mlp_class(config) |
| 136 | |
| 137 | self.config = config |
| 138 | |
| 139 | def forward( |
| 140 | self, |
nothing calls this directly
no test coverage detected