(self, config: ChatGLMConfig, layer_number, device=None)
| 768 | """ |
| 769 | |
| 770 | def __init__(self, config: ChatGLMConfig, layer_number, device=None): |
| 771 | super(GLMBlock, self).__init__() |
| 772 | self.layer_number = layer_number |
| 773 | |
| 774 | self.apply_residual_connection_post_layernorm = config.apply_residual_connection_post_layernorm |
| 775 | |
| 776 | self.fp32_residual_connection = config.fp32_residual_connection |
| 777 | |
| 778 | LayerNormFunc = RMSNorm if config.rmsnorm else LayerNorm |
| 779 | # Layernorm on the input data. |
| 780 | self.input_layernorm = LayerNormFunc(config.hidden_size, eps=config.layernorm_epsilon, device=device, |
| 781 | dtype=config.torch_dtype) |
| 782 | |
| 783 | # Self attention. |
| 784 | self.self_attention = SelfAttention(config, layer_number, device=device) |
| 785 | self.hidden_dropout = config.hidden_dropout |
| 786 | |
| 787 | # Layernorm on the attention output |
| 788 | self.post_attention_layernorm = LayerNormFunc(config.hidden_size, eps=config.layernorm_epsilon, device=device, |
| 789 | dtype=config.torch_dtype) |
| 790 | |
| 791 | # MLP |
| 792 | self.mlp = MLP(config, device=device) |
| 793 | |
| 794 | def forward( |
| 795 | self, hidden_states, attention_mask, rotary_pos_emb, kv_cache=None, use_cache=True, |
nothing calls this directly
no test coverage detected