(
self,
pretrained_path='checkpoint/music_speech_audioset_epoch_15_esc_89.98.pt',
sampling_rate=32000,
amodel = "HTSAT-base",
)
| 9 | |
| 10 | class CLAP_Encoder(nn.Module): |
| 11 | def __init__( |
| 12 | self, |
| 13 | pretrained_path='checkpoint/music_speech_audioset_epoch_15_esc_89.98.pt', |
| 14 | sampling_rate=32000, |
| 15 | amodel = "HTSAT-base", |
| 16 | ): |
| 17 | super().__init__() |
| 18 | self.device = "cpu" |
| 19 | self.precision = "fp32" |
| 20 | self.amodel = amodel # or 'PANN-14' |
| 21 | self.tmodel = "roberta" # the best text encoder in our training |
| 22 | self.enable_fusion = False # False if you do not want to use the fusion model |
| 23 | self.fusion_type = "aff_2d" |
| 24 | self.pretrained = pretrained_path |
| 25 | self.sampling_rate = sampling_rate |
| 26 | self.tokenize = RobertaTokenizer.from_pretrained("roberta-base") |
| 27 | |
| 28 | self.model, self.model_cfg = create_model( |
| 29 | self.amodel, |
| 30 | self.tmodel, |
| 31 | self.pretrained, |
| 32 | precision=self.precision, |
| 33 | device=self.device, |
| 34 | enable_fusion=self.enable_fusion, |
| 35 | fusion_type=self.fusion_type, |
| 36 | ) |
| 37 | |
| 38 | for p in self.model.parameters(): |
| 39 | p.requires_grad = False |
| 40 | |
| 41 | self.model.eval() |
| 42 | self.encoder_type = 'CLAP' |
| 43 | |
| 44 | def batch_to_list(self, batch): |
| 45 | ret = [] |
nothing calls this directly
no test coverage detected