(self, config: Gemma2Config, layer_idx: int)
| 435 | |
| 436 | class Gemma2DecoderLayer(nn.Module): |
| 437 | def __init__(self, config: Gemma2Config, layer_idx: int): |
| 438 | super().__init__() |
| 439 | self.hidden_size = config.hidden_size |
| 440 | self.config = config |
| 441 | self.is_sliding = not bool(layer_idx % 2) |
| 442 | self.self_attn = Gemma2Attention(config=config, layer_idx=layer_idx) |
| 443 | self.mlp = Gemma2MLP(config) |
| 444 | self.input_layernorm = Gemma2RMSNorm(config.hidden_size, eps=config.rms_norm_eps) |
| 445 | self.post_attention_layernorm = Gemma2RMSNorm(config.hidden_size, eps=config.rms_norm_eps) |
| 446 | |
| 447 | self.pre_feedforward_layernorm = Gemma2RMSNorm(config.hidden_size, eps=config.rms_norm_eps) |
| 448 | self.post_feedforward_layernorm = Gemma2RMSNorm(config.hidden_size, eps=config.rms_norm_eps) |
| 449 | self.sliding_window = config.sliding_window |
| 450 | |
| 451 | def forward( |
| 452 | self, |
nothing calls this directly
no test coverage detected