Multi-head attention module. Args: model_args (ModelArgs): Model configuration arguments. Attributes: n_kv_heads (int): Number of key and value heads. n_heads (int): Number of query heads. n_local_kv_heads (int): Number of local key and value heads.
| 143 | |
| 144 | |
| 145 | class Attention(nn.Module): |
| 146 | """ |
| 147 | Multi-head attention module. |
| 148 | |
| 149 | Args: |
| 150 | model_args (ModelArgs): Model configuration arguments. |
| 151 | |
| 152 | Attributes: |
| 153 | n_kv_heads (int): Number of key and value heads. |
| 154 | n_heads (int): Number of query heads. |
| 155 | n_local_kv_heads (int): Number of local key and value heads. |
| 156 | n_rep (int): Number of repetitions for local heads. |
| 157 | head_dim (int): Dimension size of each attention head. |
| 158 | wq (Linear): Linear transformation for queries. |
| 159 | wk (Linear): Linear transformation for keys. |
| 160 | wv (Linear): Linear transformation for values. |
| 161 | wo (Linear): Linear transformation for output. |
| 162 | |
| 163 | """ |
| 164 | |
| 165 | def __init__(self, model_args: ModelArgs): |
| 166 | super().__init__() |
| 167 | self.n_heads = model_args.n_heads |
| 168 | self.n_kv_heads = ( |
| 169 | model_args.n_heads |
| 170 | if model_args.n_kv_heads is None |
| 171 | else model_args.n_kv_heads |
| 172 | ) |
| 173 | self.n_rep = self.n_heads // self.n_kv_heads |
| 174 | self.head_dim = model_args.dim // model_args.n_heads |
| 175 | |
| 176 | self.wq = nn.Linear( |
| 177 | model_args.dim, model_args.n_heads * self.head_dim, bias=False |
| 178 | ) |
| 179 | self.wk = nn.Linear(model_args.dim, self.n_kv_heads * self.head_dim, bias=False) |
| 180 | self.wv = nn.Linear(model_args.dim, self.n_kv_heads * self.head_dim, bias=False) |
| 181 | self.wo = nn.Linear( |
| 182 | model_args.n_heads * self.head_dim, model_args.dim, bias=False |
| 183 | ) |
| 184 | |
| 185 | def init_weights(self, init_std: float): |
| 186 | for linear in (self.wq, self.wk, self.wv): |
| 187 | nn.init.trunc_normal_(linear.weight, mean=0.0, std=0.02) |
| 188 | nn.init.trunc_normal_(self.wo.weight, mean=0.0, std=init_std) |
| 189 | |
| 190 | def forward( |
| 191 | self, |
| 192 | x: torch.Tensor, |
| 193 | freqs_cis: torch.Tensor, |
| 194 | ): |
| 195 | """ |
| 196 | Forward pass of the attention module. |
| 197 | |
| 198 | Args: |
| 199 | x (torch.Tensor): Input tensor. |
| 200 | freqs_cis (torch.Tensor): Precomputed frequency tensor. |
| 201 | |
| 202 | Returns: |