Method get_num_params_embedding

llm_analysis/analysis.py:244–259

Get the number of parameters in the embedding layer. Args: shared_embedding (bool, optional): whether the output embedding \ shares weights with the input embedding. Defaults to True. Returns: int: the number of parameters in the embedding l

(self, shared_embedding: bool = True)

        return pivot

    def get_num_params_embedding(self, shared_embedding: bool = True) -> int:
        """Get the number of parameters in the embedding layer.

        Args:
            shared_embedding (bool, optional):  whether the output embedding \
                shares weights with the input embedding. Defaults to True.

        Returns:
            int: the number of parameters in the embedding layer
        """
        num_params_input_embedding = (self.model_config.hidden_dim *
                                      self.model_config.vocab_size)
        num_params_output_embedding = (self.model_config.hidden_dim *
                                       self.model_config.vocab_size
                                       if not shared_embedding else 0)
        return num_params_input_embedding + num_params_output_embedding

    def get_num_params_per_layer_attn(self) -> int:
        """Get the number of parameters in the attention linear layers, including the