Function generateFalAudio

apps/sim/lib/media/falai-audio.ts:84–131

(params: GenerateFalAudioParams)

}

export async function generateFalAudio(params: GenerateFalAudioParams): Promise<GeneratedAudio> {
  const type: AudioType = params.type || 'speech'
  const apiKey = getFalApiKey()

  // Voice cloning: a reference sample routes to a zero-shot clone model (F5-TTS),
  // which conditions on the sample (ref_audio_url) and speaks the prompt in that voice.
  if (params.voiceSampleDataUri) {
    const model = params.model || DEFAULT_CLONE_MODEL
    const input: Record<string, unknown> = {
      gen_text: params.prompt,
      ref_audio_url: params.voiceSampleDataUri,
      model_type: 'F5-TTS',
    }
    const { requestId, data } = await runFalQueue(model, input, apiKey)
    const url = extractFalMediaUrl(data, ['audio', 'audio_url', 'audio_file', 'output'])
    if (!url) throw new Error('No audio URL in Fal.ai clone response')
    const { buffer, contentType } = await downloadFalMedia(url)
    const cost = await getFalAICostMetadata({ apiKey, endpointId: model, requestId })
    return {
      buffer,
      contentType: contentType.startsWith('audio/') ? contentType : 'audio/mpeg',
      type: 'speech',
      model,
      jobId: requestId,
      cost,
    }
  }

  const model = params.model || DEFAULT_AUDIO_MODELS[type]
  const input = buildInput(type, params, model)

  // For fal audio models the model ID is the queue endpoint.
  const { requestId, data } = await runFalQueue(model, input, apiKey)
  const url = extractFalMediaUrl(data, ['audio', 'audio_url', 'audio_file', 'output'])
  if (!url) throw new Error('No audio URL in Fal.ai response')

  const { buffer, contentType } = await downloadFalMedia(url)
  const cost = await getFalAICostMetadata({ apiKey, endpointId: model, requestId })

  return {
    buffer,
    contentType: contentType.startsWith('audio/') ? contentType : 'audio/mpeg',
    type,
    model,
    jobId: requestId,
    cost,
  }
}