Method get_num_params_per_layer_mlp

llm_analysis/analysis.py:278–285

Get the number of parameters in the MLP linear layers, including the intermediate and output matrices. Returns: int: the number of parameters in the two MLP linear layers

(self)

            self.model_config.n_head)

    def get_num_params_per_layer_mlp(self) -> int:
        """Get the number of parameters in the MLP linear layers, including the
        intermediate and output matrices.

        Returns:
            int: the number of parameters in the two MLP linear layers
        """
        return 2 * self.model_config.hidden_dim * self.model_config.ffn_embed_dim * self.model_config.moe_num_experts

    def get_num_params_per_layer_router(self) -> int:
        if self.model_config.moe_num_experts > 1: