| 5 | |
| 6 | |
| 7 | class MLPLayers(nn.Module): |
| 8 | |
| 9 | def __init__( |
| 10 | self, layers, dropout=0.0, activation="relu", bn=False |
| 11 | ): |
| 12 | super(MLPLayers, self).__init__() |
| 13 | self.layers = layers |
| 14 | self.dropout = dropout |
| 15 | self.activation = activation |
| 16 | self.use_bn = bn |
| 17 | |
| 18 | mlp_modules = [] |
| 19 | for idx, (input_size, output_size) in enumerate( |
| 20 | zip(self.layers[:-1], self.layers[1:]) |
| 21 | ): |
| 22 | mlp_modules.append(nn.Dropout(p=self.dropout)) |
| 23 | mlp_modules.append(nn.Linear(input_size, output_size)) |
| 24 | |
| 25 | if self.use_bn and idx != (len(self.layers)-2): |
| 26 | mlp_modules.append(nn.BatchNorm1d(num_features=output_size)) |
| 27 | |
| 28 | activation_func = activation_layer(self.activation, output_size) |
| 29 | if activation_func is not None and idx != (len(self.layers)-2): |
| 30 | mlp_modules.append(activation_func) |
| 31 | |
| 32 | self.mlp_layers = nn.Sequential(*mlp_modules) |
| 33 | self.apply(self.init_weights) |
| 34 | |
| 35 | def init_weights(self, module): |
| 36 | # We just initialize the module with normal distribution as the paper said |
| 37 | if isinstance(module, nn.Linear): |
| 38 | xavier_normal_(module.weight.data) |
| 39 | if module.bias is not None: |
| 40 | module.bias.data.fill_(0.0) |
| 41 | |
| 42 | def forward(self, input_feature): |
| 43 | return self.mlp_layers(input_feature) |
| 44 | |
| 45 | def activation_layer(activation_name="relu", emb_dim=None): |
| 46 | |