| 15 | #ifdef WASMEDGE_PLUGIN_WASI_NN_BACKEND_GGML |
| 16 | |
| 17 | Expect<ErrNo> setInput(WasiNNEnvironment &Env, uint32_t ContextId, |
| 18 | uint32_t Index, const TensorData &Tensor) noexcept { |
| 19 | auto &CxtRef = Env.NNContext[ContextId].get<Context>(); |
| 20 | auto &GraphRef = Env.NNGraph[CxtRef.GraphId].get<Graph>(); |
| 21 | LOG_DEBUG(GraphRef.EnableDebugLog, "setInput"sv) |
| 22 | |
| 23 | // Use index 1 for metadata. |
| 24 | if (Index == 1) { |
| 25 | LOG_DEBUG(GraphRef.EnableDebugLog, "setInput: found Metadata, processing"sv) |
| 26 | bool IsModelParamsUpdated = false; |
| 27 | bool IsContextParamsUpdated = false; |
| 28 | bool IsSamplerParamsUpdated = false; |
| 29 | const std::string Metadata(reinterpret_cast<char *>(Tensor.Tensor.data()), |
| 30 | Tensor.Tensor.size()); |
| 31 | auto Res = |
| 32 | parseMetadata(GraphRef, CxtRef.Conf, Metadata, &IsModelParamsUpdated, |
| 33 | &IsContextParamsUpdated, &IsSamplerParamsUpdated); |
| 34 | if (Res != ErrNo::Success) { |
| 35 | RET_ERROR(Res, "setInput: failed to parse metadata."sv) |
| 36 | } |
| 37 | |
| 38 | #ifndef __APPLE__ |
| 39 | // XXX: Because of the limitation in the WASI-NN proposal, this is a |
| 40 | // workaround for non-macOS devices. However, if the model params are |
| 41 | // updated in the configuration stage, we do not recommend using this to |
| 42 | // avoid reloading the model. |
| 43 | { |
| 44 | if (IsModelParamsUpdated || GraphRef.LlamaModel == nullptr) { |
| 45 | // The llama model may be nullptr if set_input updated the model params |
| 46 | // last time. Therefore, in addition to updated model params, we should |
| 47 | // reload the llama model if the model is nullptr. |
| 48 | LOG_INFO(GraphRef.EnableLog, |
| 49 | "setInput: Reload model due to parameters change."sv) |
| 50 | llama_model_params ModelParams = llama_model_default_params(); |
| 51 | ModelParams.n_gpu_layers = |
| 52 | static_cast<int32_t>(GraphRef.Params.n_gpu_layers); |
| 53 | GraphRef.LlamaModel.reset(); |
| 54 | // Due to the model change, the context and sampler should also be |
| 55 | // reloaded. The new context and sampler will be created in the next |
| 56 | // block. |
| 57 | GraphRef.LlamaContext.reset(); |
| 58 | if (CxtRef.LlamaSampler) { |
| 59 | // TODO: Trigger the sampler in other contexts to reallocate. |
| 60 | common_sampler_free(CxtRef.LlamaSampler); |
| 61 | CxtRef.LlamaSampler = nullptr; |
| 62 | } |
| 63 | GraphRef.LlamaModel = llama_model_ptr(llama_model_load_from_file( |
| 64 | GraphRef.Params.model.path.c_str(), ModelParams)); |
| 65 | if (GraphRef.LlamaModel == nullptr) { |
| 66 | Env.NNGraph[CxtRef.GraphId].setInvalid(); |
| 67 | RET_ERROR(ErrNo::InvalidArgument, "setInput: unable to init model."sv) |
| 68 | } |
| 69 | } |
| 70 | } |
| 71 | #endif |
| 72 | |
| 73 | // Some changes to context parameters will require the context to be |
| 74 | // reloaded. |
nothing calls this directly
no test coverage detected