Method extract_spectrogram

src/common/AutoModel/modeling_gemma4e_audio.cpp:73–200

void Gemma4e::extract_spectrogram(std::vector<audio_data_t>& audio_inputs, gemma4e_audio_payload_t& audio_payload) {

    audio_payload.num_audios = static_cast<unsigned int>(audio_inputs.size());
    audio_payload.mel_spectrograms.resize(audio_payload.num_audios);
    audio_payload.mel_spectrogram_frames_per_audio.resize(audio_payload.num_audios);
    audio_payload.mel_spectrogram_bins_per_audio.resize(audio_payload.num_audios);

    // ------- Config (matches Python __init__ defaults) -------
    constexpr float frame_length_ms   = 20.0f;
    constexpr float hop_length_ms     = 10.0f;
    constexpr float min_frequency     = 0.0f;
    constexpr float max_frequency     = 8000.0f;
    constexpr float mel_floor         = 1e-3f;
    constexpr int   feature_size      = 128;     // num_mel_filters
    constexpr float dither            = 0.0f;
    constexpr float input_scale_factor = 1.0f;
    constexpr float preemphasis       = 0.0f;
    constexpr bool  preemphasis_htk_flavor = true;
    constexpr bool  fft_overdrive     = false;

    for (unsigned int audio_idx = 0; audio_idx < audio_payload.num_audios; audio_idx++) {
    audio_data_t& audio_input = audio_inputs[audio_idx];

    const int sampling_rate = audio_input.sample_rate;

    // frame_length = int(round(sampling_rate * frame_length_ms / 1000.0))
    const int frame_length = static_cast<int>(std::round(sampling_rate * frame_length_ms / 1000.0f));
    // hop_length = int(round(sampling_rate * hop_length_ms / 1000.0))
    const int hop_length   = static_cast<int>(std::round(sampling_rate * hop_length_ms / 1000.0f));

    // fft_length = 2 ** ceil(log2(frame_length))
    int fft_length = 1;
    while (fft_length < frame_length) fft_length <<= 1;
    if (fft_overdrive) fft_length *= 2;

    const int num_frequency_bins = fft_length / 2 + 1;

    // ------- self.window = window_function(frame_length).astype(np.float32) -------
    // periodic Hann window, matching Python: window_function(frame_length)
    // Python default: name="hann", periodic=True
    std::vector<float> window = audioproc::window_function_optimized(frame_length, "hann", /*periodic=*/true);

    // ------- self.mel_filters = mel_filter_bank(...) -------
    std::vector<float> mel_filters = audioproc::mel_filter_bank_optimized(
        num_frequency_bins, feature_size,
        min_frequency, max_frequency,
        sampling_rate, /*apply_slaney_norm=*/false);

    // ------- waveform = audio_input.samples (mono, 1D) -------
    // The Python code works on [B, T]. We handle B=1 (single waveform).
    const float* waveform_ptr = audio_input.samples.data();
    const int original_length = static_cast<int>(audio_input.num_frames);

    // ------- Semicausal time padding: prepend frame_length // 2 zeros -------
    // waveform = np.pad(waveform, ((0,0), (pad_left, 0)), mode="constant")
    const int pad_left = frame_length / 2;
    const int padded_length = original_length + pad_left;
    std::vector<float> waveform(padded_length, 0.0f);