| 459 | } |
| 460 | |
| 461 | bool Gemma4e::insert(chat_meta_info_t& meta_info, lm_uniform_input_t& input, std::function<bool()> is_cancelled) { |
| 462 | this->profiler_list[TKOEN_ENCODE_TIME].start(); |
| 463 | std::string templated_text; |
| 464 | |
| 465 | if (input.messages.empty() && input.prompt.empty()) { |
| 466 | header_print("WARNING", "No messages or prompt provided"); |
| 467 | return false; |
| 468 | } |
| 469 | |
| 470 | constexpr bool DEBUG_IMAGE_PREPROCESS = false; |
| 471 | gemma4e_image_payload_t image_payload; |
| 472 | gemma4e_audio_payload_t audio_payload; |
| 473 | audio_payload.num_audios = 0; |
| 474 | image_payload.num_images = 0; |
| 475 | |
| 476 | float max_support_audio_length_seconds = 30.0f; |
| 477 | int total_audio_clips = 0; |
| 478 | std::vector<audio_data_t> audio_data_list; |
| 479 | |
| 480 | if (!input.messages.empty()) { // Server Processing |
| 481 | int total_images = 0; |
| 482 | for (auto& message : input.messages) { |
| 483 | // Process Images |
| 484 | if (message.contains("images")) { |
| 485 | for (auto& img : message["images"]) { |
| 486 | std::string img_str = img.get<std::string>(); |
| 487 | if (!img_str.empty()) total_images++; |
| 488 | |
| 489 | gemma4e_image_t image = this->load_image_base64(img_str); |
| 490 | std::vector<bf16> pixel_values; |
| 491 | std::pair<int, int> patch_element_per_patch; |
| 492 | uint32_t valid_patch_size = 0; |
| 493 | uint32_t num_soft_tokens = 0; |
| 494 | std::vector<int> image_grid_pairs; |
| 495 | |
| 496 | preprocess_image(image, patch_element_per_patch, valid_patch_size, pixel_values, image_grid_pairs, num_soft_tokens); |
| 497 | |
| 498 | image_payload.image_patch__element_per_patch.push_back(patch_element_per_patch); |
| 499 | image_payload.valid_patch_size_per_image.push_back(valid_patch_size); |
| 500 | image_payload.pixel_values.push_back(pixel_values); |
| 501 | image_payload.image_grid_pairs_per_image.push_back(image_grid_pairs); |
| 502 | image_payload.num_soft_tokens_per_image.push_back(num_soft_tokens); |
| 503 | image_payload.num_images++; |
| 504 | } |
| 505 | } |
| 506 | // Process Audios |
| 507 | if (message.contains("audios")) { |
| 508 | gemma4e_npu *gemma4e_engine = dynamic_cast<gemma4e_npu*>(this->lm_engine.get()); |
| 509 | for (auto& aud : message["audios"]) { |
| 510 | std::string audio_str = aud.get<std::string>(); |
| 511 | audio_data_t audio_data = this->load_audio_base64(audio_str, gemma4e_engine->Gemma4E_Audio_resample_rate, MonoDownmixMode::MEAN); |
| 512 | if (audio_data.channels > 1) { |
| 513 | std::cerr << "only mono audio is supported." << std::endl; |
| 514 | exit(-1); |
| 515 | } |
| 516 | std::vector<audio_data_t> clipped_audio_data = this->clip_audio_length(audio_data, max_support_audio_length_seconds); |
| 517 | audio_data_list.insert(audio_data_list.end(), clipped_audio_data.begin(), clipped_audio_data.end()); |
| 518 | total_audio_clips += clipped_audio_data.size(); |
no test coverage detected