| 14 | #ifdef WASMEDGE_PLUGIN_WASI_NN_BACKEND_GGML |
| 15 | |
| 16 | Expect<ErrNo> compute(WasiNNEnvironment &Env, uint32_t ContextId) noexcept { |
| 17 | auto &CxtRef = Env.NNContext[ContextId].get<Context>(); |
| 18 | auto &GraphRef = Env.NNGraph[CxtRef.GraphId].get<Graph>(); |
| 19 | LOG_DEBUG(GraphRef.EnableDebugLog, "compute") |
| 20 | |
| 21 | // Clear the context and reset the sampler. |
| 22 | clearContext(GraphRef, CxtRef); |
| 23 | |
| 24 | if (GraphRef.Params.embedding) { |
| 25 | return getEmbedding(GraphRef, CxtRef); |
| 26 | } |
| 27 | |
| 28 | // Evaluate the input tokens. |
| 29 | ErrNo ReturnCode = ErrNo::Success; |
| 30 | if (GraphRef.VisionContext == nullptr) { |
| 31 | // Text only prompt. |
| 32 | ReturnCode = evaluateInput(GraphRef, CxtRef, "compute"sv); |
| 33 | if (ReturnCode != ErrNo::Success) { |
| 34 | return ReturnCode; |
| 35 | } |
| 36 | } else { |
| 37 | // Multimodal prompt. |
| 38 | llama_pos NewNPos; |
| 39 | int32_t Res = mtmd_helper_eval_chunks( |
| 40 | GraphRef.VisionContext.get(), GraphRef.LlamaContext.get(), |
| 41 | GraphRef.VisionInputChunks.get(), CxtRef.NPos, |
| 42 | /* seq_id */ 0, static_cast<int32_t>(CxtRef.CurrentBatchSize), |
| 43 | /* logits_last */ true, &NewNPos); |
| 44 | CxtRef.NPos = NewNPos; |
| 45 | if (Res != 0) { |
| 46 | RET_ERROR(ErrNo::InvalidArgument, |
| 47 | "compute: unable to eval the mtmd prompt."sv) |
| 48 | } |
| 49 | } |
| 50 | |
| 51 | // Main prediction loop. |
| 52 | LOG_DEBUG(GraphRef.EnableDebugLog, "compute: enter main prediction loop"sv) |
| 53 | int64_t NPredict = |
| 54 | CxtRef.Conf.NPredict < 0 ? INT32_MAX : CxtRef.Conf.NPredict; |
| 55 | |
| 56 | while (NPredict-- > 0) { |
| 57 | ReturnCode = sampleOutput(GraphRef, CxtRef); |
| 58 | if (ReturnCode != ErrNo::Success) { |
| 59 | break; |
| 60 | } |
| 61 | } |
| 62 | if (ReturnCode == ErrNo::EndOfSequence) { |
| 63 | ReturnCode = ErrNo::Success; |
| 64 | } |
| 65 | LOG_DEBUG(GraphRef.EnableDebugLog, |
| 66 | "compute: enter main prediction loop...Done"sv) |
| 67 | // End of main prediction loop. |
| 68 | |
| 69 | // TTS: convert output codes to audio file. |
| 70 | if (GraphRef.TextToSpeech) { |
| 71 | LOG_DEBUG(GraphRef.EnableDebugLog, |
| 72 | "compute: convert output codes to audio file."sv) |
| 73 | ReturnCode = codesToSpeech(Env, GraphRef, CxtRef); |
nothing calls this directly
no test coverage detected