(self, text: int["b nt"], seq_len, drop_text=False)
| 50 | self.extra_modeling = False |
| 51 | |
| 52 | def forward(self, text: int["b nt"], seq_len, drop_text=False): # noqa: F722 |
| 53 | text = text + 1 # use 0 as filler token. preprocess of batch pad -1, see list_str_to_idx() |
| 54 | text = text[:, :seq_len] # curtail if character tokens are more than the mel spec tokens |
| 55 | batch, text_len = text.shape[0], text.shape[1] |
| 56 | text = F.pad(text, (0, seq_len - text_len), value=0) |
| 57 | if self.mask_padding: |
| 58 | text_mask = text == 0 |
| 59 | |
| 60 | if drop_text: # cfg for text |
| 61 | text = torch.zeros_like(text) |
| 62 | |
| 63 | text = self.text_embed(text) # b n -> b n d |
| 64 | |
| 65 | # possible extra modeling |
| 66 | if self.extra_modeling: |
| 67 | # sinus pos emb |
| 68 | batch_start = torch.zeros((batch,), dtype=torch.long) |
| 69 | pos_idx = get_pos_embed_indices(batch_start, seq_len, max_pos=self.precompute_max_pos) |
| 70 | text_pos_embed = self.freqs_cis[pos_idx] |
| 71 | text = text + text_pos_embed |
| 72 | |
| 73 | # convnextv2 blocks |
| 74 | if self.mask_padding: |
| 75 | text = text.masked_fill(text_mask.unsqueeze(-1).expand(-1, -1, text.size(-1)), 0.0) |
| 76 | for block in self.text_blocks: |
| 77 | text = block(text) |
| 78 | text = text.masked_fill(text_mask.unsqueeze(-1).expand(-1, -1, text.size(-1)), 0.0) |
| 79 | else: |
| 80 | text = self.text_blocks(text) |
| 81 | |
| 82 | return text |
| 83 | |
| 84 | |
| 85 | # noised input audio and context mixing embedding |
nothing calls this directly
no test coverage detected