Get the number of active parameters in a transformer layer, including the attention and MoE MLP linear layers. Returns: int: the number of parameters in a transformer layer
(self)
| 310 | self.get_num_params_per_layer_layernorm()) |
| 311 | |
| 312 | def get_num_active_params_per_layer(self) -> int: |
| 313 | """Get the number of active parameters in a transformer layer, including the |
| 314 | attention and MoE MLP linear layers. |
| 315 | |
| 316 | Returns: |
| 317 | int: the number of parameters in a transformer layer |
| 318 | """ |
| 319 | |
| 320 | return ( |
| 321 | self.get_num_params_per_layer_attn() + |
| 322 | self.get_num_params_per_layer_mlp() * self.model_config.moe_top_k / |
| 323 | self.model_config.moe_num_experts + |
| 324 | self.get_num_params_per_layer_router() + |
| 325 | self.get_num_params_per_layer_layernorm()) |
| 326 | |
| 327 | def get_num_params_total(self) -> int: |
| 328 | """Get the total number of parameters in the model, including all the |
no test coverage detected