Encoder + Decoder + Calc loss Args: speech: (Batch, Length, ...) speech_lengths: (Batch, ) text: (Batch, Length) text_lengths: (Batch,)
(
self,
speech: torch.Tensor,
speech_lengths: torch.Tensor,
text: torch.Tensor,
text_lengths: torch.Tensor,
**kwargs,
)
| 213 | self.error_calculator = None |
| 214 | |
| 215 | def forward( |
| 216 | self, |
| 217 | speech: torch.Tensor, |
| 218 | speech_lengths: torch.Tensor, |
| 219 | text: torch.Tensor, |
| 220 | text_lengths: torch.Tensor, |
| 221 | **kwargs, |
| 222 | ) -> Tuple[torch.Tensor, Dict[str, torch.Tensor], torch.Tensor]: |
| 223 | """Encoder + Decoder + Calc loss |
| 224 | Args: |
| 225 | speech: (Batch, Length, ...) |
| 226 | speech_lengths: (Batch, ) |
| 227 | text: (Batch, Length) |
| 228 | text_lengths: (Batch,) |
| 229 | """ |
| 230 | if len(text_lengths.size()) > 1: |
| 231 | text_lengths = text_lengths[:, 0] |
| 232 | if len(speech_lengths.size()) > 1: |
| 233 | speech_lengths = speech_lengths[:, 0] |
| 234 | |
| 235 | batch_size = speech.shape[0] |
| 236 | |
| 237 | # Encoder |
| 238 | encoder_out, encoder_out_lens = self.encode(speech, speech_lengths) |
| 239 | |
| 240 | loss_ctc, cer_ctc = None, None |
| 241 | loss_pre = None |
| 242 | stats = dict() |
| 243 | |
| 244 | # decoder: CTC branch |
| 245 | if self.ctc_weight != 0.0: |
| 246 | loss_ctc, cer_ctc = self._calc_ctc_loss( |
| 247 | encoder_out, encoder_out_lens, text, text_lengths |
| 248 | ) |
| 249 | |
| 250 | # Collect CTC branch stats |
| 251 | stats["loss_ctc"] = loss_ctc.detach() if loss_ctc is not None else None |
| 252 | stats["cer_ctc"] = cer_ctc |
| 253 | |
| 254 | # decoder: Attention decoder branch |
| 255 | loss_att, acc_att, cer_att, wer_att, loss_pre, pre_loss_att = self._calc_att_loss( |
| 256 | encoder_out, encoder_out_lens, text, text_lengths |
| 257 | ) |
| 258 | |
| 259 | # 3. CTC-Att loss definition |
| 260 | if self.ctc_weight == 0.0: |
| 261 | loss = loss_att + loss_pre * self.predictor_weight |
| 262 | else: |
| 263 | loss = ( |
| 264 | self.ctc_weight * loss_ctc |
| 265 | + (1 - self.ctc_weight) * loss_att |
| 266 | + loss_pre * self.predictor_weight |
| 267 | ) |
| 268 | |
| 269 | # Collect Attn branch stats |
| 270 | stats["loss_att"] = loss_att.detach() if loss_att is not None else None |
| 271 | stats["pre_loss_att"] = pre_loss_att.detach() if pre_loss_att is not None else None |
| 272 | stats["acc"] = acc_att |
nothing calls this directly
no test coverage detected