(self, config: LlamaConfig, layer_idx: int)
| 709 | |
| 710 | class LlamaDecoderLayer(nn.Module): |
| 711 | def __init__(self, config: LlamaConfig, layer_idx: int): |
| 712 | super().__init__() |
| 713 | self.hidden_size = config.hidden_size |
| 714 | |
| 715 | config._attn_implementation = "longwriter" |
| 716 | self.self_attn = LLAMA_ATTENTION_CLASSES[config._attn_implementation](config=config, layer_idx=layer_idx) |
| 717 | # print(config._attn_implementation) |
| 718 | |
| 719 | self.mlp = LlamaMLP(config) |
| 720 | self.input_layernorm = LlamaRMSNorm(config.hidden_size, eps=config.rms_norm_eps) |
| 721 | self.post_attention_layernorm = LlamaRMSNorm(config.hidden_size, eps=config.rms_norm_eps) |
| 722 | |
| 723 | def forward( |
| 724 | self, |
nothing calls this directly
no test coverage detected