| 118 | |
| 119 | |
| 120 | class RelPositionalEncoding(torch.nn.Module): |
| 121 | def __init__(self, d_model, max_len=5000): |
| 122 | super().__init__() |
| 123 | pe_positive = torch.zeros(max_len, d_model, requires_grad=False) |
| 124 | pe_negative = torch.zeros(max_len, d_model, requires_grad=False) |
| 125 | position = torch.arange(0, max_len).unsqueeze(1).float() |
| 126 | div_term = torch.exp(torch.arange(0, d_model, 2).float() * |
| 127 | -(torch.log(torch.tensor(10000.0)).item()/d_model)) |
| 128 | pe_positive[:, 0::2] = torch.sin(position * div_term) |
| 129 | pe_positive[:, 1::2] = torch.cos(position * div_term) |
| 130 | pe_negative[:, 0::2] = torch.sin(-1 * position * div_term) |
| 131 | pe_negative[:, 1::2] = torch.cos(-1 * position * div_term) |
| 132 | |
| 133 | pe_positive = torch.flip(pe_positive, [0]).unsqueeze(0) |
| 134 | pe_negative = pe_negative[1:].unsqueeze(0) |
| 135 | pe = torch.cat([pe_positive, pe_negative], dim=1) |
| 136 | self.register_buffer('pe', pe) |
| 137 | |
| 138 | def forward(self, x): |
| 139 | # Tmax = 2 * max_len - 1 |
| 140 | Tmax, T = self.pe.size(1), x.size(1) |
| 141 | pos_emb = self.pe[:, Tmax // 2 - T + 1 : Tmax // 2 + T].clone().detach() |
| 142 | return pos_emb |
| 143 | |
| 144 | |
| 145 | class ConformerFeedForward(nn.Module): |