(self, config: ChatGLMConfig, layer_number, device=None)
| 510 | """ |
| 511 | |
| 512 | def __init__(self, config: ChatGLMConfig, layer_number, device=None): |
| 513 | super(GLMBlock, self).__init__() |
| 514 | self.layer_number = layer_number |
| 515 | |
| 516 | self.apply_residual_connection_post_layernorm = config.apply_residual_connection_post_layernorm |
| 517 | |
| 518 | self.fp32_residual_connection = config.fp32_residual_connection |
| 519 | |
| 520 | LayerNormFunc = RMSNorm if config.rmsnorm else LayerNorm |
| 521 | # Layernorm on the input data. |
| 522 | self.input_layernorm = LayerNormFunc(config.hidden_size, eps=config.layernorm_epsilon, device=device, |
| 523 | dtype=config.torch_dtype) |
| 524 | |
| 525 | # Self attention. |
| 526 | self.self_attention = SelfAttention(config, layer_number, device=device) |
| 527 | self.hidden_dropout = config.hidden_dropout |
| 528 | |
| 529 | # Layernorm on the attention output |
| 530 | self.post_attention_layernorm = LayerNormFunc(config.hidden_size, eps=config.layernorm_epsilon, device=device, |
| 531 | dtype=config.torch_dtype) |
| 532 | |
| 533 | # MLP |
| 534 | self.mlp = MLP(config, device=device) |
| 535 | |
| 536 | def forward( |
| 537 | self, hidden_states, attention_mask, rotary_pos_emb, kv_cache=None, use_cache=True, |
nothing calls this directly
no test coverage detected