| 333 | return x |
| 334 | |
| 335 | class MLP(nn.Module): |
| 336 | def __init__(self, in_dim=22, out_dim=1, innter_dim=96, depth=5): |
| 337 | super().__init__() |
| 338 | self.FC1 = nn.Linear(in_dim, innter_dim) |
| 339 | self.FC_out = nn.Linear(innter_dim, out_dim) |
| 340 | self.relu = torch.nn.LeakyReLU(0.2) |
| 341 | self.FC_inter = nn.ModuleList( |
| 342 | [nn.Linear(innter_dim, innter_dim) for i in range(depth)]) |
| 343 | |
| 344 | def forward(self, x): |
| 345 | x = self.FC1(x) |
| 346 | x = self.relu(x) |
| 347 | for inter_fc in self.FC_inter: |
| 348 | x = inter_fc(x) |
| 349 | x = self.relu(x) |
| 350 | x = self.FC_out(x) |
| 351 | return x |
| 352 | |
| 353 | class MultiHeadAttention(nn.Module): |
| 354 | def __init__(self, dim, heads, num_kv_tokens, cfg, rpe_bias=None, use_rpe=False): |
nothing calls this directly
no outgoing calls
no test coverage detected