Method forward

inference/models/llama.py:250–261

(
        self,
        x: torch.Tensor,
        start_pos: int,
        freqs_cis: torch.Tensor,
        mask: Optional[torch.Tensor],
    )

        self.post_attention_layernorm = RMSNorm(args.hidden_size, eps=args.rms_norm_eps)

    def forward(
        self,
        x: torch.Tensor,
        start_pos: int,
        freqs_cis: torch.Tensor,
        mask: Optional[torch.Tensor],
    ):
        h = x + self.self_attn.forward(
            self.input_layernorm(x), start_pos, freqs_cis, mask
        )
        out = h + self.mlp.forward(self.post_attention_layernorm(h))
        return out


class Transformer(nn.Module):