| 16 | from utils.ctc_alignment import ctc_forced_align |
| 17 | |
| 18 | class SinusoidalPositionEncoder(torch.nn.Module): |
| 19 | """ """ |
| 20 | |
| 21 | def __init__(self, d_model=80, dropout_rate=0.1): |
| 22 | pass |
| 23 | |
| 24 | def encode( |
| 25 | self, positions: torch.Tensor = None, depth: int = None, dtype: torch.dtype = torch.float32 |
| 26 | ): |
| 27 | batch_size = positions.size(0) |
| 28 | positions = positions.type(dtype) |
| 29 | device = positions.device |
| 30 | log_timescale_increment = torch.log(torch.tensor([10000], dtype=dtype, device=device)) / ( |
| 31 | depth / 2 - 1 |
| 32 | ) |
| 33 | inv_timescales = torch.exp( |
| 34 | torch.arange(depth / 2, device=device).type(dtype) * (-log_timescale_increment) |
| 35 | ) |
| 36 | inv_timescales = torch.reshape(inv_timescales, [batch_size, -1]) |
| 37 | scaled_time = torch.reshape(positions, [1, -1, 1]) * torch.reshape( |
| 38 | inv_timescales, [1, 1, -1] |
| 39 | ) |
| 40 | encoding = torch.cat([torch.sin(scaled_time), torch.cos(scaled_time)], dim=2) |
| 41 | return encoding.type(dtype) |
| 42 | |
| 43 | def forward(self, x): |
| 44 | batch_size, timesteps, input_dim = x.size() |
| 45 | positions = torch.arange(1, timesteps + 1, device=x.device)[None, :] |
| 46 | position_encoding = self.encode(positions, input_dim, x.dtype).to(x.device) |
| 47 | |
| 48 | return x + position_encoding |
| 49 | |
| 50 | |
| 51 | class PositionwiseFeedForward(torch.nn.Module): |