Get the number of parameters in a transformer layer, including the attention and MLP linear layers. Returns: int: the number of parameters in a transformer layer
(self)
| 297 | return self.model_config.hidden_dim |
| 298 | |
| 299 | def get_num_params_per_layer(self) -> int: |
| 300 | """Get the number of parameters in a transformer layer, including the attention |
| 301 | and MLP linear layers. |
| 302 | |
| 303 | Returns: |
| 304 | int: the number of parameters in a transformer layer |
| 305 | """ |
| 306 | |
| 307 | return (self.get_num_params_per_layer_attn() + |
| 308 | self.get_num_params_per_layer_mlp() + |
| 309 | self.get_num_params_per_layer_router() + |
| 310 | self.get_num_params_per_layer_layernorm()) |
| 311 | |
| 312 | def get_num_active_params_per_layer(self) -> int: |
| 313 | """Get the number of active parameters in a transformer layer, including the |
no test coverage detected