(
self,
hidden_states: torch.Tensor,
hidden_length: torch.Tensor,
apply_position: bool = True,
)
| 232 | self.embed_positions.weight.copy_(sinusoids(*self.embed_positions.weight.shape)) |
| 233 | |
| 234 | def forward( |
| 235 | self, |
| 236 | hidden_states: torch.Tensor, |
| 237 | hidden_length: torch.Tensor, |
| 238 | apply_position: bool = True, |
| 239 | ): |
| 240 | # Downsampling |
| 241 | hidden_states = hidden_states.transpose(1, 2) |
| 242 | hidden_states = F.gelu(self.conv1(hidden_states)) |
| 243 | hidden_states = F.gelu(self.conv2(hidden_states)) |
| 244 | hidden_states = hidden_states.transpose(1, 2) |
| 245 | hidden_length = hidden_length // 2 # from 100Hz -> 50Hz |
| 246 | # Pos encoding |
| 247 | if apply_position: |
| 248 | pos_embed = self.embed_positions( |
| 249 | torch.arange(0, hidden_states.shape[1], device=hidden_states.device) |
| 250 | ) |
| 251 | hidden_states = hidden_states + pos_embed |
| 252 | hidden_states = nn.functional.dropout( |
| 253 | hidden_states, p=self.dropout, training=self.training |
| 254 | ) |
| 255 | # Transformer |
| 256 | attention_mask = make_nonpad_mask(hidden_length).unsqueeze(1) # (b, 1, t) |
| 257 | for layer in self.layers: |
| 258 | hidden_states = layer(hidden_states, attention_mask) |
| 259 | |
| 260 | hidden_states = self.layer_norm(hidden_states) |
| 261 | return hidden_states, hidden_length |
| 262 | |
| 263 | def _init_weights(self, module): |
| 264 | std = 0.02 |
no test coverage detected