Class FunAudioChatPlugin

training/plugin/mm_plugins.py:77–276

FunAudioChat multimodal plugin for audio, image and video processing.

@dataclass
class FunAudioChatPlugin(BasePlugin):
    """
    FunAudioChat multimodal plugin for audio, image and video processing.
    """
    audio_reader: str = "read_audio"  # Can be "read_audio" or custom reader

    @staticmethod
    def load_audio_batch(
        paths: list[str], sampling_rate: int, audio_reader: str = "read_audio"
    ) -> list[np.ndarray]:
        """
        Loads a batch of audio files.
        
        Args:
            paths: List of audio file paths
            sampling_rate: Target sampling rate
            audio_reader: Audio reader type (default: "read_audio")
            
        Returns:
            List of audio arrays
        """
        return load_audio_batch_default(paths, sampling_rate)

    @override
    def _get_mm_inputs(
        self,
        images: list["ImageInput"],
        videos: list["VideoInput"],
        audios: list["AudioInput"],
        processor: "MMProcessor",
        imglens: Optional[list[int]] = None,
    ) -> dict[str, "torch.Tensor"]:
        """Process multimodal inputs including audio."""
        
        # Load audio files if they are paths
        if len(audios) > 0 and (isinstance(audios[0], str) or isinstance(audios[0], list)):
            audios = FunAudioChatPlugin.load_audio_batch(
                audios, 
                sampling_rate=getattr(processor, "audio_sampling_rate", 16000),
                audio_reader=self.audio_reader
            )

        mm_inputs = {}
        
        # Process audios
        if len(audios) != 0:
            feature_extractor = getattr(processor, "feature_extractor", None)
            audios = self._regularize_audios(
                audios,
                sampling_rate=getattr(processor, "audio_sampling_rate", 16000),
            )["audios"]
            
            with torch.inference_mode():
                mm_inputs.update(
                    feature_extractor(
                        audios,
                        sampling_rate=getattr(processor, "audio_sampling_rate", 16000),
                        return_attention_mask=True,