Parse metadata from JSON.
| 14 | #ifdef WASMEDGE_PLUGIN_WASI_NN_BACKEND_GGML |
| 15 | // Parse metadata from JSON. |
| 16 | ErrNo parseMetadata(Graph &GraphRef, LocalConfig &ConfRef, |
| 17 | const std::string &Metadata, bool *IsModelUpdated, |
| 18 | bool *IsContextUpdated, bool *IsSamplerUpdated) noexcept { |
| 19 | // Parse metadata from the json. |
| 20 | simdjson::dom::parser Parser; |
| 21 | simdjson::dom::element Doc; |
| 22 | auto ParseError = Parser.parse(Metadata).get(Doc); |
| 23 | if (ParseError) { |
| 24 | RET_ERROR(ErrNo::InvalidEncoding, "parse metadata error."sv) |
| 25 | } |
| 26 | |
| 27 | // Get the current llama parameters. |
| 28 | int64_t PrevNGPULayers = GraphRef.Params.n_gpu_layers; |
| 29 | bool PrevEmbedding = GraphRef.Params.embedding; |
| 30 | // Get the current sampler parameters. |
| 31 | double PrevTemp = GraphRef.Params.sampling.temp; |
| 32 | double PrevTopP = GraphRef.Params.sampling.top_p; |
| 33 | double PrevRepeatPenalty = GraphRef.Params.sampling.penalty_repeat; |
| 34 | double PrevPresencePenalty = GraphRef.Params.sampling.penalty_present; |
| 35 | double PrevFrequencyPenalty = GraphRef.Params.sampling.penalty_freq; |
| 36 | std::string PrevGrammar = |
| 37 | common_grammar_value(GraphRef.Params.sampling.grammar); |
| 38 | uint32_t PrevSeed = GraphRef.Params.sampling.seed; |
| 39 | |
| 40 | try { |
| 41 | parseJsonAuto(Doc, "enable-log", GraphRef.EnableLog); |
| 42 | parseJsonAuto(Doc, "enable-debug-log", GraphRef.EnableDebugLog); |
| 43 | |
| 44 | parseJsonWithCastAuto<int64_t>(Doc, "main-gpu", GraphRef.Params.main_gpu); |
| 45 | parseJsonWithCastAuto<int64_t>(Doc, "n-gpu-layers", |
| 46 | GraphRef.Params.n_gpu_layers); |
| 47 | |
| 48 | parseJsonWithProcessorAuto<bool>(Doc, "cpu-moe", |
| 49 | [&GraphRef](const bool &CpuMoe) -> bool { |
| 50 | if (CpuMoe) { |
| 51 | GraphRef.TensorBuftOverrides.push_back( |
| 52 | "\\.ffn_(up|down|gate)_exps"); |
| 53 | } |
| 54 | return true; |
| 55 | }); |
| 56 | |
| 57 | parseJsonWithProcessorAuto<int64_t>( |
| 58 | Doc, "n-cpu-moe", [&GraphRef](const int64_t &NCpuMoe) -> bool { |
| 59 | if (NCpuMoe < 0) { |
| 60 | spdlog::error("[WASI-NN] GGML backend: Invalid n-cpu-moe value."); |
| 61 | return false; |
| 62 | } |
| 63 | for (int I = 0; I < NCpuMoe; I++) { |
| 64 | GraphRef.TensorBuftOverrides.push_back( |
| 65 | string_format("blk\\.%d\\.ffn_(up|down|gate)_exps", I)); |
| 66 | } |
| 67 | return true; |
| 68 | }); |
| 69 | parseJsonWithProcessorAuto<std::string_view>( |
| 70 | Doc, "tensor-split", [&GraphRef](const std::string_view &TSV) -> bool { |
| 71 | // The TensorSplit is a comma-separated list of non-negative values. |
| 72 | // E.g., "3,2" presents 60% of the data to GPU 0 and 40% to GPU 1. |
| 73 | std::string TS(TSV); |