Reference: Highway Networks. For now we don't limit the type of the gate and forward. Caller should init Highway with transformer and carry and guarantee the dim to be matching.
| 148 | |
| 149 | |
| 150 | class Highway(torch.nn.Module): |
| 151 | """ |
| 152 | Reference: Highway Networks. |
| 153 | For now we don't limit the type of the gate and forward. |
| 154 | Caller should init Highway with transformer and carry and guarantee the dim |
| 155 | to be matching. |
| 156 | """ |
| 157 | |
| 158 | def __init__(self, transformer_gate, transformer_forward): |
| 159 | super(Highway, self).__init__() |
| 160 | self.transformer_forward = transformer_forward |
| 161 | self.transformer_gate = transformer_gate |
| 162 | |
| 163 | def forward(self, x, gate_input=None, forward_input=None): |
| 164 | if gate_input is None: |
| 165 | gate_input = x |
| 166 | if forward_input is None: |
| 167 | forward_input = x |
| 168 | gate = self.transformer_gate(gate_input) |
| 169 | forward = self.transformer_forward(forward_input) |
| 170 | return gate * forward + (1 - gate) * x |