| 52 | } |
| 53 | |
| 54 | bool Gemma3::insert(chat_meta_info_t& meta_info, lm_uniform_input_t& input, std::function<bool()> is_cancelled) { |
| 55 | static constexpr int IMAGE_TOKEN_ID = 262144; // replace with actual image token ID |
| 56 | this->profiler_list[TKOEN_ENCODE_TIME].start(); |
| 57 | std::string templated_text; |
| 58 | if (input.messages.empty() && input.prompt.empty()) { |
| 59 | header_print("WARNING", "No messages or prompt provided"); |
| 60 | return false; |
| 61 | } |
| 62 | if (!input.messages.empty()) { // already a formated messages, usually from REST API |
| 63 | templated_text = this->apply_chat_template(input.messages); |
| 64 | // std::cout << "Templated text after applying chat template: \n" << templated_text << std::endl; // Debug print |
| 65 | } |
| 66 | else if (!input.prompt.empty()) { // a pure text, usually from the cli |
| 67 | nlohmann::ordered_json messages; |
| 68 | if (input.images.size() > 0) { |
| 69 | nlohmann::ordered_json content; |
| 70 | content["role"] = "user"; |
| 71 | content["content"] = input.prompt; |
| 72 | content["images"] = nlohmann::ordered_json::array(); |
| 73 | for (int i = 0; i < input.images.size(); i++) { |
| 74 | content["images"].push_back(input.images[i]); |
| 75 | } |
| 76 | messages.push_back(content); |
| 77 | } |
| 78 | else { |
| 79 | messages.push_back({ {"role", "user"}, {"content", input.prompt} }); |
| 80 | } |
| 81 | templated_text = this->apply_chat_template(messages); |
| 82 | } |
| 83 | // process all images |
| 84 | |
| 85 | bytes pixel_values; |
| 86 | if (!input.messages.empty()) { |
| 87 | int total_images = 0; |
| 88 | for (auto& message : input.messages){ |
| 89 | nlohmann::ordered_json::array_t images = message.value("images", nlohmann::ordered_json::array()); |
| 90 | if (images.size() > 0){ |
| 91 | total_images += images.size(); |
| 92 | } |
| 93 | } |
| 94 | header_print("FLM", "Total images: " << total_images); |
| 95 | // temporary solution |
| 96 | if (total_images > 0){ |
| 97 | pixel_values.resize(3 * 896 * 896 * sizeof(bf16) * total_images); |
| 98 | uint8_t* pixel_values_ptr = pixel_values.data(); |
| 99 | for (auto& message : input.messages){ |
| 100 | nlohmann::ordered_json::array_t images = message.value("images", nlohmann::ordered_json::array()); |
| 101 | for (auto& image : images){ |
| 102 | std::string image_str = image.get<std::string>(); |
| 103 | bytes image_rgb = load_image_base64(image_str); |
| 104 | buffer<bf16> pv = preprocess_image(image_rgb); |
| 105 | memcpy(pixel_values_ptr, pv.data(), pv.size() * sizeof(bf16)); |
| 106 | pixel_values_ptr += pv.size() * sizeof(bf16); |
| 107 | } |
| 108 | } |
| 109 | } |
| 110 | } |
| 111 | else { // from cli, typically only one image, typically a file path |
no test coverage detected