r""" Constructs a FunAudioChat processor which wraps a FunAudioChat feature extractor and a FunAudioChat tokenizer into a single processor. [`FunAudioChatProcessor`] offers all the functionalities of [`WhisperFeatureExtractor`] and [`FunAudioChatTokenizerFast`]. See the [`~FunAudioChatP
| 57 | |
| 58 | |
| 59 | class FunAudioChatProcessor(ProcessorMixin): |
| 60 | r""" |
| 61 | Constructs a FunAudioChat processor which wraps a FunAudioChat feature extractor and a FunAudioChat tokenizer into a single processor. |
| 62 | |
| 63 | [`FunAudioChatProcessor`] offers all the functionalities of [`WhisperFeatureExtractor`] and [`FunAudioChatTokenizerFast`]. See the |
| 64 | [`~FunAudioChatProcessor.__call__`] and [`~FunAudioChatProcessor.decode`] for more information. |
| 65 | |
| 66 | Args: |
| 67 | feature_extractor ([`WhisperFeatureExtractor`], *optional*): |
| 68 | The feature extractor is a required input. |
| 69 | tokenizer ([`FunAudioChatTokenizerFast`], *optional*): |
| 70 | The tokenizer is a required input. |
| 71 | chat_template (`Optional[str]`, *optional*): |
| 72 | The Jinja template to use for formatting the conversation. If not provided, the default chat template |
| 73 | is used. |
| 74 | audio_token (`str`, *optional*, defaults to `"<|AUDIO|>"`): |
| 75 | The token to use for audio tokens. |
| 76 | audio_bos_token (`str`, *optional*, defaults to `"<|audio_bos|>"`): |
| 77 | The token to use for audio bos tokens. |
| 78 | audio_eos_token (`str`, *optional*, defaults to `"<|audio_eos|>"`): |
| 79 | The token to use for audio eos tokens. |
| 80 | """ |
| 81 | |
| 82 | attributes = ["feature_extractor", "speech_tokenizer", "tokenizer"] |
| 83 | valid_kwargs = ["chat_template", "audio_token", "audio_bos_token", "audio_eos_token", "audio_pad_token", "audio_group_size"] |
| 84 | feature_extractor_class = "WhisperFeatureExtractor" |
| 85 | speech_tokenizer_class = "AutoTokenizer" |
| 86 | tokenizer_class = "AutoTokenizer" |
| 87 | |
| 88 | def __init__( |
| 89 | self, |
| 90 | feature_extractor=None, |
| 91 | speech_tokenizer=None, |
| 92 | tokenizer=None, |
| 93 | chat_template=None, |
| 94 | audio_token="<|AUDIO|>", |
| 95 | audio_bos_token="<|audio_bos|>", |
| 96 | audio_eos_token="<|audio_eos|>", |
| 97 | audio_pad_token="<|audio_pad|>", |
| 98 | audio_group_size=5, |
| 99 | ): |
| 100 | if chat_template is None: |
| 101 | chat_template = self.default_chat_template |
| 102 | self.audio_token = tokenizer.audio_token if hasattr(tokenizer, "audio_token") else audio_token |
| 103 | self.audio_token_id = tokenizer.convert_tokens_to_ids(self.audio_token) |
| 104 | self.audio_bos_token = tokenizer.audio_bos_token if hasattr(tokenizer, "audio_bos_token") else audio_bos_token |
| 105 | self.audio_eos_token = tokenizer.audio_eos_token if hasattr(tokenizer, "audio_eos_token") else audio_eos_token |
| 106 | self.audio_pad_token = tokenizer.audio_pad_token if hasattr(tokenizer, "audio_pad_token") else audio_pad_token |
| 107 | if speech_tokenizer is not None: |
| 108 | self.audio_group_size = speech_tokenizer.init_kwargs.get("audio_group_size", audio_group_size) |
| 109 | super().__init__(feature_extractor, speech_tokenizer, tokenizer, chat_template=chat_template) |
| 110 | |
| 111 | def _regularize_audios( |
| 112 | self, audios: list["AudioInput"], sampling_rate: float, **kwargs |
| 113 | ) -> dict[str, Union[list["NDArray"], list[float]]]: |
| 114 | r"""Regularizes audios to avoid error. Including reading and resampling.""" |
| 115 | results, sampling_rates = [], [] |
| 116 | for audio in audios: |
nothing calls this directly
no outgoing calls
no test coverage detected