Frontend + Encoder. Note that this method is used by asr_inference.py Args: speech: (Batch, Length, ...) speech_lengths: (Batch, )
(
self, speech: torch.Tensor, speech_lengths: torch.Tensor
)
| 378 | return {"feats": feats, "feats_lengths": feats_lengths} |
| 379 | |
| 380 | def encode( |
| 381 | self, speech: torch.Tensor, speech_lengths: torch.Tensor |
| 382 | ) -> Tuple[torch.Tensor, torch.Tensor]: |
| 383 | """Frontend + Encoder. Note that this method is used by asr_inference.py |
| 384 | |
| 385 | Args: |
| 386 | speech: (Batch, Length, ...) |
| 387 | speech_lengths: (Batch, ) |
| 388 | """ |
| 389 | with autocast("cuda", enabled=self.autocast_frontend, dtype=autocast_type): |
| 390 | # 1. Extract feats |
| 391 | feats, feats_lengths = self._extract_feats(speech, speech_lengths) |
| 392 | |
| 393 | # 2. Data augmentation |
| 394 | if self.specaug is not None and self.training: |
| 395 | feats, feats_lengths = self.specaug(feats, feats_lengths) |
| 396 | |
| 397 | # 3. Normalization for feature: e.g. Global-CMVN, Utterance-CMVN |
| 398 | if self.normalize is not None: |
| 399 | feats, feats_lengths = self.normalize(feats, feats_lengths) |
| 400 | |
| 401 | # Pre-encoder, e.g. used for raw input data |
| 402 | if self.preencoder is not None: |
| 403 | feats, feats_lengths = self.preencoder(feats, feats_lengths) |
| 404 | |
| 405 | # 4. Forward encoder |
| 406 | # feats: (Batch, Length, Dim) |
| 407 | # -> encoder_out: (Batch, Length2, Dim2) |
| 408 | if self.encoder is None: |
| 409 | encoder_out, encoder_out_lens = feats, feats_lengths |
| 410 | else: |
| 411 | |
| 412 | if getattr(self.encoder, "interctc_use_conditioning", False) or getattr( |
| 413 | self.encoder, "ctc_trim", False |
| 414 | ): |
| 415 | encoder_out, encoder_out_lens, _ = self.encoder( |
| 416 | feats, feats_lengths, ctc=self.ctc |
| 417 | ) |
| 418 | else: |
| 419 | encoder_out, encoder_out_lens, _ = self.encoder(feats, feats_lengths) |
| 420 | |
| 421 | intermediate_outs = None |
| 422 | if isinstance(encoder_out, tuple): |
| 423 | intermediate_outs = encoder_out[1] |
| 424 | encoder_out = encoder_out[0] |
| 425 | |
| 426 | # Post-encoder, e.g. NLU |
| 427 | if self.postencoder is not None: |
| 428 | encoder_out, encoder_out_lens = self.postencoder( |
| 429 | encoder_out, encoder_out_lens |
| 430 | ) |
| 431 | |
| 432 | assert encoder_out.size(0) == speech.size(0), ( |
| 433 | encoder_out.size(), |
| 434 | speech.size(0), |
| 435 | ) |
| 436 | if self.encoder is not None and ( |
| 437 | getattr(self.encoder, "selfattention_layer_type", None) != "lf_selfattn" |
no test coverage detected