| 8 | |
| 9 | |
| 10 | class CLAP_Encoder(nn.Module): |
| 11 | def __init__( |
| 12 | self, |
| 13 | pretrained_path='checkpoint/music_speech_audioset_epoch_15_esc_89.98.pt', |
| 14 | sampling_rate=32000, |
| 15 | amodel = "HTSAT-base", |
| 16 | ): |
| 17 | super().__init__() |
| 18 | self.device = "cpu" |
| 19 | self.precision = "fp32" |
| 20 | self.amodel = amodel # or 'PANN-14' |
| 21 | self.tmodel = "roberta" # the best text encoder in our training |
| 22 | self.enable_fusion = False # False if you do not want to use the fusion model |
| 23 | self.fusion_type = "aff_2d" |
| 24 | self.pretrained = pretrained_path |
| 25 | self.sampling_rate = sampling_rate |
| 26 | self.tokenize = RobertaTokenizer.from_pretrained("roberta-base") |
| 27 | |
| 28 | self.model, self.model_cfg = create_model( |
| 29 | self.amodel, |
| 30 | self.tmodel, |
| 31 | self.pretrained, |
| 32 | precision=self.precision, |
| 33 | device=self.device, |
| 34 | enable_fusion=self.enable_fusion, |
| 35 | fusion_type=self.fusion_type, |
| 36 | ) |
| 37 | |
| 38 | for p in self.model.parameters(): |
| 39 | p.requires_grad = False |
| 40 | |
| 41 | self.model.eval() |
| 42 | self.encoder_type = 'CLAP' |
| 43 | |
| 44 | def batch_to_list(self, batch): |
| 45 | ret = [] |
| 46 | for i in range(batch.size(0)): |
| 47 | ret.append(batch[i]) |
| 48 | return ret |
| 49 | |
| 50 | def _get_audio_embed(self, batch): |
| 51 | # batch: [B, samples] |
| 52 | with torch.no_grad(): |
| 53 | audio_dict_list = [] |
| 54 | assert ( |
| 55 | self.sampling_rate == 32000 |
| 56 | ), "We only support 32000 sampling rate" |
| 57 | |
| 58 | # batch: [bs, 1, t-samples] |
| 59 | batch = torchaudio.functional.resample( |
| 60 | batch, orig_freq=self.sampling_rate, new_freq=48000 |
| 61 | ) |
| 62 | for waveform in self.batch_to_list(batch): |
| 63 | audio_dict = {} |
| 64 | audio_dict = get_audio_features( |
| 65 | audio_dict, |
| 66 | waveform, |
| 67 | 480000, |
no outgoing calls
no test coverage detected