(self, config: ChatGLMConfig, layer_number, device=None)
| 420 | """ |
| 421 | |
| 422 | def __init__(self, config: ChatGLMConfig, layer_number, device=None): |
| 423 | super(GLMBlock, self).__init__() |
| 424 | self.layer_number = layer_number |
| 425 | |
| 426 | self.apply_residual_connection_post_layernorm = config.apply_residual_connection_post_layernorm |
| 427 | |
| 428 | self.fp32_residual_connection = config.fp32_residual_connection |
| 429 | |
| 430 | LayerNormFunc = RMSNorm if config.rmsnorm else LayerNorm |
| 431 | # Layernorm on the input data. |
| 432 | self.input_layernorm = LayerNormFunc(config.hidden_size, eps=config.layernorm_epsilon, device=device, |
| 433 | dtype=config.torch_dtype) |
| 434 | |
| 435 | # Self attention. |
| 436 | self.self_attention = SelfAttention(config, layer_number, device=device) |
| 437 | self.hidden_dropout = config.hidden_dropout |
| 438 | |
| 439 | # Layernorm on the attention output |
| 440 | self.post_attention_layernorm = LayerNormFunc(config.hidden_size, eps=config.layernorm_epsilon, device=device, |
| 441 | dtype=config.torch_dtype) |
| 442 | |
| 443 | # MLP |
| 444 | self.mlp = MLP(config, device=device) |
| 445 | |
| 446 | def forward( |
| 447 | self, hidden_states, attention_mask, rotary_pos_emb, kv_cache=None, use_cache=True, |
nothing calls this directly
no test coverage detected