Method __init__

inference/models/mpt.py:223–228

(self, d_model: int, expansion_ratio: int)

class MPTMLP(nn.Module):
    def __init__(self, d_model: int, expansion_ratio: int):
        super().__init__()
        self.up_proj = nn.Linear(d_model, expansion_ratio * d_model, bias=False)
        self.act = nn.GELU(approximate="none")
        self.down_proj = nn.Linear(expansion_ratio * d_model, d_model, bias=False)
        self.down_proj._is_residual = True

    def forward(self, x):
        return self.down_proj(self.act(self.up_proj(x)))