\brief Constructor \param supported_models - the list of supported models \param downloader - the downloader for the models \param tag - the tag of the model to load
| 37 | /// \param downloader - the downloader for the models |
| 38 | /// \param tag - the tag of the model to load |
| 39 | Runner::Runner(model_list& supported_models, ModelDownloader& downloader, program_args_t& args) |
| 40 | : supported_models(supported_models), downloader(downloader), tag(args.model_tag), asr(args.asr), embed(args.embed), img_pre_resize(args.img_pre_resize), preemption(args.preemption) { |
| 41 | |
| 42 | this->npu_device_inst = xrt::device(0); |
| 43 | |
| 44 | if (args.ctx_length != -1) { |
| 45 | this->ctx_length = args.ctx_length >= 512 ? args.ctx_length : 512; |
| 46 | } |
| 47 | else { |
| 48 | this->ctx_length = -1; |
| 49 | } |
| 50 | |
| 51 | if (this->auto_chat_engine != nullptr) { |
| 52 | this->auto_chat_engine.reset(); |
| 53 | } |
| 54 | std::pair<std::string, std::unique_ptr<AutoModel>> auto_model = get_auto_model(this->tag, this->supported_models, &this->npu_device_inst); |
| 55 | this->auto_chat_engine = std::move(auto_model.second); |
| 56 | |
| 57 | this->tag = auto_model.first; |
| 58 | |
| 59 | if (!this->downloader.is_model_downloaded(this->tag)) { |
| 60 | this->downloader.pull_model(this->tag); |
| 61 | } |
| 62 | auto [new_tag, model_info] = this->supported_models.get_model_info(this->tag); |
| 63 | this->asr_supported = model_info.contains("asr") && model_info["asr"]; |
| 64 | // header_print("ASR", asr_supported); |
| 65 | this->auto_chat_engine->configure_parameter("img_pre_resize", this->img_pre_resize); |
| 66 | try { |
| 67 | this->auto_chat_engine->load_model(this->supported_models.get_model_path(new_tag), model_info, this->ctx_length, this->preemption); |
| 68 | } |
| 69 | catch (const std::exception& e) { |
| 70 | header_print("ERROR", "Failed to load model: " + std::string(e.what())); |
| 71 | exit(EXIT_FAILURE); |
| 72 | } |
| 73 | |
| 74 | try { |
| 75 | if (args.prefill_chunk_len != -1) { |
| 76 | this->prefill_chunk_len = args.prefill_chunk_len >= 512 ? args.prefill_chunk_len : 512; |
| 77 | } |
| 78 | else { |
| 79 | this->prefill_chunk_len = model_info["max_prefill_len"].get<int>();; |
| 80 | } |
| 81 | } |
| 82 | catch (const std::exception& e) { |
| 83 | this->prefill_chunk_len = 4096; // default value |
| 84 | } |
| 85 | |
| 86 | this->generate_limit = -1; |
| 87 | |
| 88 | |
| 89 | if (this->embed) { |
| 90 | header_print("Warning", "Embed model not supported in CLI; Use 'flm serve -e 1'"); |
| 91 | } |
| 92 | |
| 93 | header_print("FLM", "Loading model: " << this->tag); |
| 94 | |
| 95 | #ifndef FASTFLOWLM_LINUX_LIMITED_MODELS |
| 96 | if (this->asr) { |
nothing calls this directly
no test coverage detected