| 507 | } |
| 508 | |
| 509 | bool mtmd_audio_preprocessor_whisper::preprocess(const float * samples, |
| 510 | size_t n_samples, |
| 511 | std::vector<mtmd_audio_mel> & output) { |
| 512 | if (n_samples == 0) { |
| 513 | // empty audio |
| 514 | return false; |
| 515 | } |
| 516 | |
| 517 | std::vector<float> smpl; |
| 518 | // if input is too short, pad with zeros |
| 519 | // this is to avoid potential issues with stage1/2 padding in log_mel_spectrogram |
| 520 | // TODO: maybe handle this better |
| 521 | size_t min_samples = (size_t) hparams.audio_sample_rate * (hparams.audio_chunk_len + 1); // +1 second margin |
| 522 | if (n_samples < min_samples) { |
| 523 | smpl.resize(min_samples, 0.0f); |
| 524 | std::memcpy(smpl.data(), samples, n_samples * sizeof(float)); |
| 525 | samples = smpl.data(); |
| 526 | n_samples = smpl.size(); |
| 527 | } |
| 528 | |
| 529 | filter_params params; |
| 530 | params.n_mel = hparams.n_mel_bins; |
| 531 | params.n_fft_bins = 1 + (hparams.audio_n_fft / 2); |
| 532 | params.hann_window_size = hparams.audio_window_len; |
| 533 | params.hop_length = hparams.audio_hop_len; |
| 534 | params.sample_rate = hparams.audio_sample_rate; |
| 535 | params.center_padding = false; |
| 536 | params.preemph = 0.0f; // disabled |
| 537 | params.use_natural_log = false; |
| 538 | params.norm_per_feature = false; |
| 539 | |
| 540 | // make sure the cache is initialized |
| 541 | GGML_ASSERT(!cache.sin_vals.empty()); |
| 542 | GGML_ASSERT(!cache.cos_vals.empty()); |
| 543 | GGML_ASSERT(!cache.filters.data.empty()); |
| 544 | |
| 545 | mtmd_audio_mel out_full; |
| 546 | bool ok = log_mel_spectrogram(samples, n_samples, |
| 547 | 4, // n_threads |
| 548 | params, cache, out_full); |
| 549 | if (!ok) { |
| 550 | return false; |
| 551 | } |
| 552 | |
| 553 | // because the cgraph in clip.cpp only accepts 3000 frames each, we need to split the mel |
| 554 | // we always expect the mel to have 3000 silent frames at the end |
| 555 | if (DEBUG) { |
| 556 | printf("output: n_mel = %d, n_len = %d\n", out_full.n_mel, out_full.n_len); |
| 557 | } |
| 558 | const size_t frames_per_chunk = 3000; |
| 559 | GGML_ASSERT((size_t) out_full.n_len > frames_per_chunk); |
| 560 | for (size_t off = 0; off < (size_t) out_full.n_len; off += frames_per_chunk) { |
| 561 | int n_len = std::min(frames_per_chunk, (size_t) out_full.n_len - off); |
| 562 | if ((size_t) n_len < frames_per_chunk) { |
| 563 | break; // last uncomplete chunk will always be a padded chunk, safe to ignore |
| 564 | } |
| 565 | |
| 566 | mtmd_audio_mel out_chunk; |