(self, x)
| 342 | [nn.Linear(innter_dim, innter_dim) for i in range(depth)]) |
| 343 | |
| 344 | def forward(self, x): |
| 345 | x = self.FC1(x) |
| 346 | x = self.relu(x) |
| 347 | for inter_fc in self.FC_inter: |
| 348 | x = inter_fc(x) |
| 349 | x = self.relu(x) |
| 350 | x = self.FC_out(x) |
| 351 | return x |
| 352 | |
| 353 | class MultiHeadAttention(nn.Module): |
| 354 | def __init__(self, dim, heads, num_kv_tokens, cfg, rpe_bias=None, use_rpe=False): |
nothing calls this directly
no outgoing calls
no test coverage detected