(self, d_model, nhead, bidirectional=True, dropout=0, activation="relu")
| 35 | """ |
| 36 | |
| 37 | def __init__(self, d_model, nhead, bidirectional=True, dropout=0, activation="relu"): |
| 38 | super(TransformerEncoderLayer, self).__init__() |
| 39 | |
| 40 | |
| 41 | #self.self_attn = PerformerSelfAttention(d_model, nhead) |
| 42 | # Implementation of Feedforward model |
| 43 | #dim_feedforward = d_model * 2 |
| 44 | #self.linear1 = Linear(d_model, dim_feedforward) |
| 45 | self.gru = GRU(d_model, d_model*2, 1, bidirectional=bidirectional) |
| 46 | self.dropout = Dropout(dropout) |
| 47 | #self.linear2 = Linear(dim_feedforward, d_model) |
| 48 | |
| 49 | if bidirectional: |
| 50 | self.linear2 = Linear(d_model*2*2, d_model) |
| 51 | self.self_attn = MultiheadAttention(d_model, nhead, dropout=dropout) |
| 52 | self.norm1 = InstantLayerNorm1d(d_model) |
| 53 | self.dropout1 = Dropout(dropout) |
| 54 | else: |
| 55 | self.linear2 = Linear(d_model*2, d_model) |
| 56 | |
| 57 | self.norm2 = InstantLayerNorm1d(d_model) |
| 58 | self.dropout2 = Dropout(dropout) |
| 59 | |
| 60 | self.activation = _get_activation_fn(activation) |
| 61 | self.bidirectional = bidirectional |
| 62 | |
| 63 | def __setstate__(self, state): |
| 64 | if 'activation' not in state: |
nothing calls this directly
no test coverage detected