| 34 | |
| 35 | |
| 36 | class GroupWiseLinear(nn.Module): |
| 37 | def __init__(self, num_class, input_dim, output_dim, bias=True): |
| 38 | super().__init__() |
| 39 | self.num_class = num_class |
| 40 | self.input_dim = input_dim |
| 41 | self.output_dim = output_dim |
| 42 | self.bias = bias |
| 43 | |
| 44 | self.W = nn.Parameter(torch.Tensor(num_class, input_dim, output_dim)) |
| 45 | if bias: |
| 46 | self.b = nn.Parameter(torch.Tensor(num_class, output_dim)) |
| 47 | self.reset_parameters() |
| 48 | |
| 49 | def reset_parameters(self): |
| 50 | stdv = 1. / math.sqrt(self.W.size(2)) |
| 51 | for i in range(self.num_class): |
| 52 | for j in range(self.input_dim): |
| 53 | self.W[i][j].data.uniform_(-stdv, stdv) |
| 54 | if self.bias: |
| 55 | for i in range(self.num_class): |
| 56 | self.b[i].data.uniform_(-stdv, stdv) |
| 57 | |
| 58 | def forward(self, x: torch.FloatTensor): |
| 59 | """ |
| 60 | |
| 61 | Dim: |
| 62 | - b: batch size |
| 63 | - k: num_class |
| 64 | - d: input dim |
| 65 | - o: output dim |
| 66 | |
| 67 | Input: |
| 68 | - x: shape(b,k,d) or (c0,b,k,d) |
| 69 | |
| 70 | Output: |
| 71 | - x: shape(b,k,o) or (c0,b,k,o) |
| 72 | """ |
| 73 | if x.dim() == 4: |
| 74 | resize_flag = True |
| 75 | c0, b, k, d = x.shape |
| 76 | x = x.flatten(0, 1) |
| 77 | else: |
| 78 | resize_flag = False |
| 79 | |
| 80 | x = torch.einsum('bkd,kdo->bko', x, self.W) |
| 81 | if self.bias: |
| 82 | x = torch.einsum('bko,ko->bko', x, self.b) |
| 83 | |
| 84 | if resize_flag: |
| 85 | x = x.reshape(c0, b, k, -1) |
| 86 | return x |
| 87 | |
| 88 | |
| 89 | class MLP(nn.Module): |