Method get_num_params_per_layer_router

llm_analysis/analysis.py:287–291

(self)

        return 2 * self.model_config.hidden_dim * self.model_config.ffn_embed_dim * self.model_config.moe_num_experts

    def get_num_params_per_layer_router(self) -> int:
        if self.model_config.moe_num_experts > 1:
            return self.model_config.hidden_dim * self.model_config.moe_num_experts
        else:
            return 0

    def get_num_params_per_layer_layernorm(self) -> int:
        return 2 * self.model_config.hidden_dim