Method __init__

(
            self,
            dropout: float = 0.0,
            bos_token_id: int = 0,
            eos_token_id: int = 1,
            hidden_act: str = "silu",
            hidden_size: int = 512,
            intermediate_size: int = None,
            max_position_embeddings: int = 32768,
            num_attention_heads: int = 8,
            num_hidden_layers: int = 8,
            num_key_value_heads: int = 2,
            vocab_size: int = 6400,
            rms_norm_eps: float = 1e-5,
            rope_theta: int = 1000000.0,
            inference_rope_scaling: bool = False,
            flash_attn: bool = True,

            ########################################
            # Here are the specific configurations of MOE
            # When use moe is false, the following is invalid
            #########################################
            use_moe: bool = False,
            num_experts_per_tok: int = 2,
            n_router_experts: int = 4,
            n_shared_experts: int = 1,
            scoring_func: str = "softmax",
            aux_loss_alpha: float = 0.1,
            seq_aux: bool = True,
            norm_topk_prob: bool = True,
            **kwargs,
    )