Function decode_audio

src-tauri/src/transcriber.rs:314–380

Decode any media for whisper

(media_data: Arc<[u8]>)

/// Decode any media for whisper
fn decode_audio(media_data: Arc<[u8]>) -> anyhow::Result<Vec<f32>> {
    use gst::prelude::*;

    let media_data_len = media_data.len();
    let media_data = Arc::new(Mutex::new(Cursor::new(media_data)));

    let pipeline = 
        gst::parse::launch(&format!("appsrc name=audio-in stream-type=2 size={media_data_len} ! decodebin ! audioconvert ! audio/x-raw,format=F32LE,channels=1 ! audioresample ! audio/x-raw,rate=16000 ! appsink name=pcm-out sync=false"))?
            .dynamic_cast::<gst::Pipeline>().unwrap();

    pipeline.by_name("audio-in").unwrap().dynamic_cast::<gst_app::AppSrc>().unwrap()
        .set_callbacks(gst_app::AppSrcCallbacks::builder()
            .need_data({
                let media_data = media_data.clone();

                move |source, length| {
                    let mut buffer = vec![0u8; length as _];
                    let position = media_data.lock().unwrap().position();
                    let _ = media_data.lock().unwrap().read(&mut buffer).unwrap();

                    if position as usize >= media_data_len {
                        println!("Ending stream");
                        source.end_of_stream().unwrap();
                        return;
                    }

                    let buffer = gst::Buffer::from_slice(buffer);
                    let _ = source.push_buffer(buffer);
                }
            })
            .seek_data({
                let media_data = media_data.clone();

                move |_, location| {
                    media_data.lock().unwrap().set_position(location);

                    true
                }
            })
            .build());

    let pcm_data = Arc::new(Mutex::new(Vec::new()));

    pipeline.by_name("pcm-out").unwrap().dynamic_cast::<gst_app::AppSink>().unwrap()
        .set_callbacks(gst_app::AppSinkCallbacks::builder()
            .new_sample({
                let pcm_data = pcm_data.clone();

                move |sink| {
                    let Ok(sample) = sink.pull_sample() else { return Err(gst::FlowError::Error) };

                    let buffer = sample.buffer().unwrap();
                    let mapped_buffer = buffer.map_readable().unwrap();

                    let samples = mapped_buffer.as_slice_of::<f32>().unwrap();

                    pcm_data.lock().unwrap().extend_from_slice(samples);