(self, config: TelechatConfig, layer_idx)
| 507 | |
| 508 | class TelechatBlock(nn.Module): |
| 509 | def __init__(self, config: TelechatConfig, layer_idx): |
| 510 | super().__init__() |
| 511 | hidden_size = config.hidden_size |
| 512 | |
| 513 | self.input_layernorm = MixedFusedRMSNorm(hidden_size, eps=config.layer_norm_epsilon) |
| 514 | self.num_heads = config.n_head |
| 515 | self.layer_idx = layer_idx |
| 516 | self.self_attention = TelechatAttention(config, layer_idx) |
| 517 | self.post_attention_layernorm = MixedFusedRMSNorm(hidden_size, eps=config.layer_norm_epsilon) |
| 518 | |
| 519 | self.mlp = TelechatMLP(config) |
| 520 | |
| 521 | self.apply_residual_connection_post_layernorm = config.apply_residual_connection_post_layernorm |
| 522 | self.hidden_dropout = config.hidden_dropout |
| 523 | |
| 524 | def forward( |
| 525 | self, |
nothing calls this directly
no test coverage detected